mirror of
https://github.com/ysoftdevs/terraform-aws-eks.git
synced 2026-03-25 10:51:06 +01:00
feat: Add support for Pod Identity assocation on Karpenter sub-module (#3031)
Co-authored-by: Bryant Biggs <bryantbiggs@gmail.com>
This commit is contained in:
Rodrigo Bersa
@@ -1,6 +1,6 @@
|
||||
# Karpenter Example
|
||||
|
||||
Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on EKS Fargate yet Karpenter is providing compute in the form of EC2 instances.
|
||||
Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is provisioned on top of an EKS Managed Node Group.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -22,10 +22,47 @@ aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter
|
||||
kubectl scale deployment inflate --replicas 5
|
||||
|
||||
# You can watch Karpenter's controller logs with
|
||||
kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller
|
||||
kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller
|
||||
```
|
||||
|
||||
You should see a new node named `karpenter.sh/provisioner-name/default` eventually come up in the console; this was provisioned by Karpenter in response to the scaled deployment above.
|
||||
Validate if the Amazon EKS Addons Pods are running in the Managed Node Group and the `inflate` application Pods are running on Karpenter provisioned Nodes.
|
||||
|
||||
```bash
|
||||
kubectl get nodes -L karpenter.sh/registered
|
||||
```
|
||||
|
||||
```text
|
||||
NAME STATUS ROLES AGE VERSION REGISTERED
|
||||
ip-10-0-16-155.eu-west-1.compute.internal Ready <none> 100s v1.29.3-eks-ae9a62a true
|
||||
ip-10-0-3-23.eu-west-1.compute.internal Ready <none> 6m1s v1.29.3-eks-ae9a62a
|
||||
ip-10-0-41-2.eu-west-1.compute.internal Ready <none> 6m3s v1.29.3-eks-ae9a62a
|
||||
```
|
||||
|
||||
```sh
|
||||
kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName
|
||||
```
|
||||
|
||||
```text
|
||||
NAME NODE
|
||||
inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal
|
||||
inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal
|
||||
inflate-75d744d4c6-sp4dx ip-10-0-16-155.eu-west-1.compute.internal
|
||||
inflate-75d744d4c6-xqzd9 ip-10-0-16-155.eu-west-1.compute.internal
|
||||
inflate-75d744d4c6-xr6p5 ip-10-0-16-155.eu-west-1.compute.internal
|
||||
aws-node-mnn7r ip-10-0-3-23.eu-west-1.compute.internal
|
||||
aws-node-rkmvm ip-10-0-16-155.eu-west-1.compute.internal
|
||||
aws-node-s4slh ip-10-0-41-2.eu-west-1.compute.internal
|
||||
coredns-68bd859788-7rcfq ip-10-0-3-23.eu-west-1.compute.internal
|
||||
coredns-68bd859788-l78hw ip-10-0-41-2.eu-west-1.compute.internal
|
||||
eks-pod-identity-agent-gbx8l ip-10-0-41-2.eu-west-1.compute.internal
|
||||
eks-pod-identity-agent-s7vt7 ip-10-0-16-155.eu-west-1.compute.internal
|
||||
eks-pod-identity-agent-xwgqw ip-10-0-3-23.eu-west-1.compute.internal
|
||||
karpenter-79f59bdfdc-9q5ff ip-10-0-41-2.eu-west-1.compute.internal
|
||||
karpenter-79f59bdfdc-cxvhr ip-10-0-3-23.eu-west-1.compute.internal
|
||||
kube-proxy-7crbl ip-10-0-41-2.eu-west-1.compute.internal
|
||||
kube-proxy-jtzds ip-10-0-16-155.eu-west-1.compute.internal
|
||||
kube-proxy-sm42c ip-10-0-3-23.eu-west-1.compute.internal
|
||||
```
|
||||
|
||||
### Tear Down & Clean-Up
|
||||
|
||||
|
||||
@@ -41,9 +41,8 @@ data "aws_ecrpublic_authorization_token" "token" {
|
||||
}
|
||||
|
||||
locals {
|
||||
name = "ex-${replace(basename(path.cwd), "_", "-")}"
|
||||
cluster_version = "1.29"
|
||||
region = "eu-west-1"
|
||||
name = "ex-${basename(path.cwd)}"
|
||||
region = "eu-west-1"
|
||||
|
||||
vpc_cidr = "10.0.0.0/16"
|
||||
azs = slice(data.aws_availability_zones.available.names, 0, 3)
|
||||
@@ -62,62 +61,42 @@ locals {
|
||||
module "eks" {
|
||||
source = "../.."
|
||||
|
||||
cluster_name = local.name
|
||||
cluster_version = local.cluster_version
|
||||
cluster_endpoint_public_access = true
|
||||
cluster_name = local.name
|
||||
cluster_version = "1.29"
|
||||
|
||||
# Gives Terraform identity admin access to cluster which will
|
||||
# allow deploying resources (Karpenter) into the cluster
|
||||
enable_cluster_creator_admin_permissions = true
|
||||
cluster_endpoint_public_access = true
|
||||
|
||||
cluster_addons = {
|
||||
coredns = {
|
||||
configuration_values = jsonencode({
|
||||
computeType = "Fargate"
|
||||
# Ensure that we fully utilize the minimum amount of resources that are supplied by
|
||||
# Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html
|
||||
# Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes
|
||||
# components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following
|
||||
# compute configuration that most closely matches the sum of vCPU and memory requests in
|
||||
# order to ensure pods always have the resources that they need to run.
|
||||
resources = {
|
||||
limits = {
|
||||
cpu = "0.25"
|
||||
# We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the
|
||||
# request/limit to ensure we can fit within that task
|
||||
memory = "256M"
|
||||
}
|
||||
requests = {
|
||||
cpu = "0.25"
|
||||
# We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the
|
||||
# request/limit to ensure we can fit within that task
|
||||
memory = "256M"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
kube-proxy = {}
|
||||
vpc-cni = {}
|
||||
coredns = {}
|
||||
eks-pod-identity-agent = {}
|
||||
kube-proxy = {}
|
||||
vpc-cni = {}
|
||||
}
|
||||
|
||||
vpc_id = module.vpc.vpc_id
|
||||
subnet_ids = module.vpc.private_subnets
|
||||
control_plane_subnet_ids = module.vpc.intra_subnets
|
||||
|
||||
# Fargate profiles use the cluster primary security group so these are not utilized
|
||||
create_cluster_security_group = false
|
||||
create_node_security_group = false
|
||||
|
||||
fargate_profiles = {
|
||||
eks_managed_node_groups = {
|
||||
karpenter = {
|
||||
selectors = [
|
||||
{ namespace = "karpenter" }
|
||||
]
|
||||
}
|
||||
kube-system = {
|
||||
selectors = [
|
||||
{ namespace = "kube-system" }
|
||||
]
|
||||
instance_types = ["m5.large"]
|
||||
|
||||
min_size = 2
|
||||
max_size = 3
|
||||
desired_size = 2
|
||||
|
||||
taints = {
|
||||
# This Taint aims to keep just EKS Addons and Karpenter running on this MNG
|
||||
# The pods that do not tolerate this taint should run on nodes created by Karpenter
|
||||
addons = {
|
||||
key = "CriticalAddonsOnly"
|
||||
value = "true"
|
||||
effect = "NO_SCHEDULE"
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,9 +117,8 @@ module "karpenter" {
|
||||
|
||||
cluster_name = module.eks.cluster_name
|
||||
|
||||
# EKS Fargate currently does not support Pod Identity
|
||||
enable_irsa = true
|
||||
irsa_oidc_provider_arn = module.eks.oidc_provider_arn
|
||||
enable_pod_identity = true
|
||||
create_pod_identity_association = true
|
||||
|
||||
# Used to attach additional IAM policies to the Karpenter node IAM role
|
||||
node_iam_role_additional_policies = {
|
||||
@@ -162,14 +140,13 @@ module "karpenter_disabled" {
|
||||
################################################################################
|
||||
|
||||
resource "helm_release" "karpenter" {
|
||||
namespace = "karpenter"
|
||||
create_namespace = true
|
||||
namespace = "kube-system"
|
||||
name = "karpenter"
|
||||
repository = "oci://public.ecr.aws/karpenter"
|
||||
repository_username = data.aws_ecrpublic_authorization_token.token.user_name
|
||||
repository_password = data.aws_ecrpublic_authorization_token.token.password
|
||||
chart = "karpenter"
|
||||
version = "0.35.1"
|
||||
version = "0.36.1"
|
||||
wait = false
|
||||
|
||||
values = [
|
||||
@@ -178,14 +155,6 @@ resource "helm_release" "karpenter" {
|
||||
clusterName: ${module.eks.cluster_name}
|
||||
clusterEndpoint: ${module.eks.cluster_endpoint}
|
||||
interruptionQueue: ${module.karpenter.queue_name}
|
||||
serviceAccount:
|
||||
annotations:
|
||||
eks.amazonaws.com/role-arn: ${module.karpenter.iam_role_arn}
|
||||
tolerations:
|
||||
- key: 'eks.amazonaws.com/compute-type'
|
||||
operator: Equal
|
||||
value: fargate
|
||||
effect: "NoSchedule"
|
||||
EOT
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user