feat: New Karpenter sub-module for easily enabling Karpenter on EKS (#2303)

This commit is contained in:
Bryant Biggs
2022-11-21 13:50:34 -05:00
committed by GitHub
parent 1bc86e19aa
commit f24de3326d
28 changed files with 1336 additions and 162 deletions

View File

@@ -29,6 +29,7 @@ Please note that we strive to provide a comprehensive suite of documentation for
- [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html)
- [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html)
- [Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html)
- Support for creating Karpenter related AWS infrastruture resources (e.g. IAM roles, SQS queue, EventBridge rules, etc.)
- Support for custom AMI, custom launch template, and custom user data including custom user data template
- Support for Amazon Linux 2 EKS Optimized AMI and Bottlerocket nodes
- Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes
@@ -60,7 +61,7 @@ module "eks" {
version = "~> 18.0"
cluster_name = "my-cluster"
cluster_version = "1.22"
cluster_version = "1.24"
cluster_endpoint_private_access = true
cluster_endpoint_public_access = true
@@ -295,7 +296,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
| <a name="input_cluster_service_ipv4_cidr"></a> [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no |
| <a name="input_cluster_tags"></a> [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no |
| <a name="input_cluster_timeouts"></a> [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no |
| <a name="input_cluster_version"></a> [cluster\_version](#input\_cluster\_version) | Kubernetes `<major>.<minor>` version to use for the EKS cluster (i.e.: `1.22`) | `string` | `null` | no |
| <a name="input_cluster_version"></a> [cluster\_version](#input\_cluster\_version) | Kubernetes `<major>.<minor>` version to use for the EKS cluster (i.e.: `1.24`) | `string` | `null` | no |
| <a name="input_control_plane_subnet_ids"></a> [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no |
| <a name="input_create"></a> [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no |
| <a name="input_create_aws_auth_configmap"></a> [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no |
@@ -363,8 +364,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
| <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
| <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
| <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
| <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
| <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console |

View File

@@ -141,9 +141,9 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com
1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default:
```hcl
cluster_version = "1.22"
cluster_version = "1.24"
# This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.22
# This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.24
self_managed_node_groups = {
default = {}
}
@@ -152,7 +152,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com
2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI:
```hcl
cluster_version = "1.22"
cluster_version = "1.24"
self_managed_node_groups = {
bottlerocket = {

View File

@@ -8,7 +8,7 @@ module "eks" {
source = "terraform-aws-modules/eks/aws"
cluster_name = "example"
cluster_version = "1.22"
cluster_version = "1.24"
cluster_addons = {
vpc-cni = {

View File

@@ -81,8 +81,9 @@ No inputs.
| <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
| <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
| <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
| <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
| <a name="output_cluster_security_group_arn"></a> [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group |

View File

@@ -10,7 +10,7 @@ provider "kubernetes" {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
@@ -267,7 +267,7 @@ module "eks_managed_node_group" {
source = "../../modules/eks-managed-node-group"
name = "separate-eks-mng"
cluster_name = module.eks.cluster_id
cluster_name = module.eks.cluster_name
cluster_version = module.eks.cluster_version
vpc_id = module.vpc.vpc_id
@@ -298,7 +298,7 @@ module "self_managed_node_group" {
source = "../../modules/self-managed-node-group"
name = "separate-self-mng"
cluster_name = module.eks.cluster_id
cluster_name = module.eks.cluster_name
cluster_version = module.eks.cluster_version
cluster_endpoint = module.eks.cluster_endpoint
cluster_auth_base64 = module.eks.cluster_certificate_authority_data
@@ -319,7 +319,7 @@ module "fargate_profile" {
source = "../../modules/fargate-profile"
name = "separate-fargate-profile"
cluster_name = module.eks.cluster_id
cluster_name = module.eks.cluster_name
subnet_ids = module.vpc.private_subnets
selectors = [{

View File

@@ -17,8 +17,13 @@ output "cluster_endpoint" {
value = module.eks.cluster_endpoint
}
output "cluster_name" {
description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_name
}
output "cluster_id" {
description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_id
}

View File

@@ -115,8 +115,9 @@ No inputs.
| <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
| <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
| <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
| <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
| <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console |

View File

@@ -10,13 +10,13 @@ provider "kubernetes" {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.22"
cluster_version = "1.24"
region = "eu-west-1"
tags = {

View File

@@ -17,8 +17,13 @@ output "cluster_endpoint" {
value = module.eks.cluster_endpoint
}
output "cluster_name" {
description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_name
}
output "cluster_id" {
description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_id
}

View File

@@ -68,8 +68,9 @@ No inputs.
| <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
| <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
| <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
| <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
| <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console |

View File

@@ -11,14 +11,14 @@ provider "helm" {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
}
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.22"
cluster_version = "1.24"
region = "eu-west-1"
tags = {
@@ -104,7 +104,7 @@ module "eks" {
################################################################################
data "aws_eks_cluster_auth" "this" {
name = module.eks.cluster_id
name = module.eks.cluster_name
}
locals {
@@ -113,7 +113,7 @@ locals {
kind = "Config"
current-context = "terraform"
clusters = [{
name = module.eks.cluster_id
name = module.eks.cluster_name
cluster = {
certificate-authority-data = module.eks.cluster_certificate_authority_data
server = module.eks.cluster_endpoint
@@ -122,7 +122,7 @@ locals {
contexts = [{
name = "terraform"
context = {
cluster = module.eks.cluster_id
cluster = module.eks.cluster_name
user = "terraform"
}
}]

View File

@@ -17,8 +17,13 @@ output "cluster_endpoint" {
value = module.eks.cluster_endpoint
}
output "cluster_name" {
description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_name
}
output "cluster_id" {
description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_id
}

View File

@@ -1,6 +1,6 @@
# Karpenter Example
Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling.
Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on EKS Fargate yet Karpenter is providing compute in the form of EC2 instances.
## Usage
@@ -55,6 +55,8 @@ Note that this example may create resources which cost money. Run `terraform des
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.4 |
| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14 |
| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
| <a name="requirement_null"></a> [null](#requirement\_null) | >= 3.0 |
## Providers
@@ -63,24 +65,29 @@ Note that this example may create resources which cost money. Run `terraform des
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
| <a name="provider_helm"></a> [helm](#provider\_helm) | >= 2.4 |
| <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | >= 1.14 |
| <a name="provider_null"></a> [null](#provider\_null) | >= 3.0 |
## Modules
| Name | Source | Version |
|------|--------|---------|
| <a name="module_eks"></a> [eks](#module\_eks) | ../.. | n/a |
| <a name="module_karpenter_irsa"></a> [karpenter\_irsa](#module\_karpenter\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.21.1 |
| <a name="module_karpenter"></a> [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a |
| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 |
## Resources
| Name | Type |
|------|------|
| [aws_iam_instance_profile.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
| [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
| [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
| [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
| [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source |
| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
## Inputs
@@ -100,8 +107,9 @@ No inputs.
| <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
| <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
| <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
| <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
| <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console |
@@ -112,6 +120,20 @@ No inputs.
| <a name="output_eks_managed_node_groups"></a> [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created |
| <a name="output_eks_managed_node_groups_autoscaling_group_names"></a> [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups |
| <a name="output_fargate_profiles"></a> [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created |
| <a name="output_karpenter_event_rules"></a> [karpenter\_event\_rules](#output\_karpenter\_event\_rules) | Map of the event rules created and their attributes |
| <a name="output_karpenter_instance_profile_arn"></a> [karpenter\_instance\_profile\_arn](#output\_karpenter\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile |
| <a name="output_karpenter_instance_profile_id"></a> [karpenter\_instance\_profile\_id](#output\_karpenter\_instance\_profile\_id) | Instance profile's ID |
| <a name="output_karpenter_instance_profile_name"></a> [karpenter\_instance\_profile\_name](#output\_karpenter\_instance\_profile\_name) | Name of the instance profile |
| <a name="output_karpenter_instance_profile_unique"></a> [karpenter\_instance\_profile\_unique](#output\_karpenter\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile |
| <a name="output_karpenter_irsa_arn"></a> [karpenter\_irsa\_arn](#output\_karpenter\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts |
| <a name="output_karpenter_irsa_name"></a> [karpenter\_irsa\_name](#output\_karpenter\_irsa\_name) | The name of the IAM role for service accounts |
| <a name="output_karpenter_irsa_unique_id"></a> [karpenter\_irsa\_unique\_id](#output\_karpenter\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts |
| <a name="output_karpenter_queue_arn"></a> [karpenter\_queue\_arn](#output\_karpenter\_queue\_arn) | The ARN of the SQS queue |
| <a name="output_karpenter_queue_name"></a> [karpenter\_queue\_name](#output\_karpenter\_queue\_name) | The name of the created Amazon SQS queue |
| <a name="output_karpenter_queue_url"></a> [karpenter\_queue\_url](#output\_karpenter\_queue\_url) | The URL for the created Amazon SQS queue |
| <a name="output_karpenter_role_arn"></a> [karpenter\_role\_arn](#output\_karpenter\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role |
| <a name="output_karpenter_role_name"></a> [karpenter\_role\_name](#output\_karpenter\_role\_name) | The name of the IAM role |
| <a name="output_karpenter_role_unique_id"></a> [karpenter\_role\_unique\_id](#output\_karpenter\_role\_unique\_id) | Stable and unique string identifying the IAM role |
| <a name="output_node_security_group_arn"></a> [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group |
| <a name="output_node_security_group_id"></a> [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group |
| <a name="output_oidc_provider"></a> [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) |

View File

@@ -2,13 +2,50 @@ provider "aws" {
region = local.region
}
data "aws_partition" "current" {}
provider "kubernetes" {
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
provider "helm" {
kubernetes {
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
}
provider "kubectl" {
apply_retry_count = 5
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
load_config_file = false
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.22"
cluster_version = "1.24"
region = "eu-west-1"
partition = data.aws_partition.current.partition
tags = {
Example = local.name
@@ -32,29 +69,35 @@ module "eks" {
vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.private_subnets
node_security_group_additional_rules = {
# Control plane invoke Karpenter webhook
ingress_karpenter_webhook_tcp = {
description = "Control plane invoke Karpenter webhook"
protocol = "tcp"
from_port = 8443
to_port = 8443
type = "ingress"
source_cluster_security_group = true
# Fargate profiles use the cluster primary security group so these are not utilized
create_cluster_security_group = false
create_node_security_group = false
manage_aws_auth_configmap = true
aws_auth_roles = [
# We need to add in the Karpenter node IAM role for nodes launched by Karpenter
{
rolearn = module.karpenter.role_arn
username = "system:node:{{EC2PrivateDNSName}}"
groups = [
"system:bootstrappers",
"system:nodes",
]
},
]
fargate_profiles = {
kube_system = {
name = "kube-system"
selectors = [
{ namespace = "kube-system" }
]
}
}
eks_managed_node_groups = {
karpenter = {
instance_types = ["t3.medium"]
min_size = 1
max_size = 2
desired_size = 1
iam_role_additional_policies = [
# Required by Karpenter
"arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore"
name = "karpenter"
selectors = [
{ namespace = "karpenter" }
]
}
}
@@ -71,60 +114,14 @@ module "eks" {
# Karpenter
################################################################################
provider "helm" {
kubernetes {
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
}
}
}
module "karpenter" {
source = "../../modules/karpenter"
provider "kubectl" {
apply_retry_count = 5
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
load_config_file = false
cluster_name = module.eks.cluster_name
irsa_oidc_provider_arn = module.eks.oidc_provider_arn
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
}
}
module "karpenter_irsa" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 4.21.1"
role_name = "karpenter-controller-${local.name}"
attach_karpenter_controller_policy = true
karpenter_controller_cluster_id = module.eks.cluster_id
karpenter_controller_ssm_parameter_arns = [
"arn:${local.partition}:ssm:*:*:parameter/aws/service/*"
]
karpenter_controller_node_iam_role_arns = [
module.eks.eks_managed_node_groups["karpenter"].iam_role_arn
]
oidc_providers = {
ex = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["karpenter:karpenter"]
}
}
}
resource "aws_iam_instance_profile" "karpenter" {
name = "KarpenterNodeInstanceProfile-${local.name}"
role = module.eks.eks_managed_node_groups["karpenter"].iam_role_name
tags = local.tags
}
resource "helm_release" "karpenter" {
@@ -132,54 +129,73 @@ resource "helm_release" "karpenter" {
create_namespace = true
name = "karpenter"
repository = "https://charts.karpenter.sh"
repository = "oci://public.ecr.aws/karpenter"
chart = "karpenter"
version = "0.8.2"
version = "v0.19.1"
set {
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = module.karpenter_irsa.iam_role_arn
name = "settings.aws.clusterName"
value = module.eks.cluster_name
}
set {
name = "clusterName"
value = module.eks.cluster_id
}
set {
name = "clusterEndpoint"
name = "settings.aws.clusterEndpoint"
value = module.eks.cluster_endpoint
}
set {
name = "aws.defaultInstanceProfile"
value = aws_iam_instance_profile.karpenter.name
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = module.karpenter.irsa_arn
}
set {
name = "settings.aws.defaultInstanceProfile"
value = module.karpenter.instance_profile_name
}
set {
name = "settings.aws.interruptionQueueName"
value = module.karpenter.queue_name
}
}
# Workaround - https://github.com/hashicorp/terraform-provider-kubernetes/issues/1380#issuecomment-967022975
resource "kubectl_manifest" "karpenter_provisioner" {
yaml_body = <<-YAML
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
name: default
spec:
requirements:
- key: karpenter.sh/capacity-type
operator: In
values: ["spot"]
limits:
resources:
cpu: 1000
provider:
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
name: default
spec:
requirements:
- key: karpenter.sh/capacity-type
operator: In
values: ["spot"]
limits:
resources:
cpu: 1000
providerRef:
name: default
ttlSecondsAfterEmpty: 30
YAML
depends_on = [
helm_release.karpenter
]
}
resource "kubectl_manifest" "karpenter_node_template" {
yaml_body = <<-YAML
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
name: default
spec:
subnetSelector:
karpenter.sh/discovery: ${local.name}
karpenter.sh/discovery: ${module.eks.cluster_name}
securityGroupSelector:
karpenter.sh/discovery: ${local.name}
karpenter.sh/discovery: ${module.eks.cluster_name}
tags:
karpenter.sh/discovery: ${local.name}
ttlSecondsAfterEmpty: 30
karpenter.sh/discovery: ${module.eks.cluster_name}
YAML
depends_on = [
@@ -191,27 +207,27 @@ resource "kubectl_manifest" "karpenter_provisioner" {
# and starts with zero replicas
resource "kubectl_manifest" "karpenter_example_deployment" {
yaml_body = <<-YAML
apiVersion: apps/v1
kind: Deployment
metadata:
name: inflate
spec:
replicas: 0
selector:
matchLabels:
app: inflate
template:
metadata:
labels:
apiVersion: apps/v1
kind: Deployment
metadata:
name: inflate
spec:
replicas: 0
selector:
matchLabels:
app: inflate
spec:
terminationGracePeriodSeconds: 0
containers:
- name: inflate
image: public.ecr.aws/eks-distro/kubernetes/pause:3.2
resources:
requests:
cpu: 1
template:
metadata:
labels:
app: inflate
spec:
terminationGracePeriodSeconds: 0
containers:
- name: inflate
image: public.ecr.aws/eks-distro/kubernetes/pause:3.7
resources:
requests:
cpu: 1
YAML
depends_on = [
@@ -219,6 +235,129 @@ resource "kubectl_manifest" "karpenter_example_deployment" {
]
}
################################################################################
# Modify EKS CoreDNS Deployment
################################################################################
data "aws_eks_cluster_auth" "this" {
name = module.eks.cluster_name
}
locals {
kubeconfig = yamlencode({
apiVersion = "v1"
kind = "Config"
current-context = "terraform"
clusters = [{
name = module.eks.cluster_name
cluster = {
certificate-authority-data = module.eks.cluster_certificate_authority_data
server = module.eks.cluster_endpoint
}
}]
contexts = [{
name = "terraform"
context = {
cluster = module.eks.cluster_name
user = "terraform"
}
}]
users = [{
name = "terraform"
user = {
token = data.aws_eks_cluster_auth.this.token
}
}]
})
}
# Separate resource so that this is only ever executed once
resource "null_resource" "remove_default_coredns_deployment" {
triggers = {}
provisioner "local-exec" {
interpreter = ["/bin/bash", "-c"]
environment = {
KUBECONFIG = base64encode(local.kubeconfig)
}
# We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon
# However, we are maintaing the existing kube-dns service and annotating it for Helm to assume control
command = <<-EOT
kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode)
EOT
}
}
resource "null_resource" "modify_kube_dns" {
triggers = {}
provisioner "local-exec" {
interpreter = ["/bin/bash", "-c"]
environment = {
KUBECONFIG = base64encode(local.kubeconfig)
}
# We are maintaing the existing kube-dns service and annotating it for Helm to assume control
command = <<-EOT
echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}"
kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode)
kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-namespace=kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode)
kubectl --namespace kube-system label --overwrite service kube-dns app.kubernetes.io/managed-by=Helm --kubeconfig <(echo $KUBECONFIG | base64 --decode)
EOT
}
depends_on = [
null_resource.remove_default_coredns_deployment
]
}
################################################################################
# CoreDNS Helm Chart (self-managed)
################################################################################
data "aws_eks_addon_version" "this" {
for_each = toset(["coredns"])
addon_name = each.value
kubernetes_version = module.eks.cluster_version
most_recent = true
}
resource "helm_release" "coredns" {
name = "coredns"
namespace = "kube-system"
create_namespace = false
description = "CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS Services"
chart = "coredns"
version = "1.19.4"
repository = "https://coredns.github.io/helm"
# For EKS image repositories https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html
values = [
<<-EOT
image:
repository: 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns
tag: ${data.aws_eks_addon_version.this["coredns"].version}
deployment:
name: coredns
annotations:
eks.amazonaws.com/compute-type: fargate
service:
name: kube-dns
annotations:
eks.amazonaws.com/compute-type: fargate
podAnnotations:
eks.amazonaws.com/compute-type: fargate
EOT
]
depends_on = [
# Need to ensure the CoreDNS updates are peformed before provisioning
null_resource.modify_kube_dns
]
}
################################################################################
# Supporting Resources
################################################################################

View File

@@ -17,8 +17,13 @@ output "cluster_endpoint" {
value = module.eks.cluster_endpoint
}
output "cluster_name" {
description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_name
}
output "cluster_id" {
description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_id
}
@@ -185,3 +190,93 @@ output "aws_auth_configmap_yaml" {
description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles"
value = module.eks.aws_auth_configmap_yaml
}
################################################################################
# IAM Role for Service Account (IRSA)
################################################################################
output "karpenter_irsa_name" {
description = "The name of the IAM role for service accounts"
value = module.karpenter.irsa_name
}
output "karpenter_irsa_arn" {
description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts"
value = module.karpenter.irsa_arn
}
output "karpenter_irsa_unique_id" {
description = "Stable and unique string identifying the IAM role for service accounts"
value = module.karpenter.irsa_unique_id
}
################################################################################
# Node Termination Queue
################################################################################
output "karpenter_queue_arn" {
description = "The ARN of the SQS queue"
value = module.karpenter.queue_arn
}
output "karpenter_queue_name" {
description = "The name of the created Amazon SQS queue"
value = module.karpenter.queue_name
}
output "karpenter_queue_url" {
description = "The URL for the created Amazon SQS queue"
value = module.karpenter.queue_url
}
################################################################################
# Node Termination Event Rules
################################################################################
output "karpenter_event_rules" {
description = "Map of the event rules created and their attributes"
value = module.karpenter.event_rules
}
################################################################################
# Node IAM Role
################################################################################
output "karpenter_role_name" {
description = "The name of the IAM role"
value = module.karpenter.role_name
}
output "karpenter_role_arn" {
description = "The Amazon Resource Name (ARN) specifying the IAM role"
value = module.karpenter.role_arn
}
output "karpenter_role_unique_id" {
description = "Stable and unique string identifying the IAM role"
value = module.karpenter.role_unique_id
}
################################################################################
# Node IAM Instance Profile
################################################################################
output "karpenter_instance_profile_arn" {
description = "ARN assigned by AWS to the instance profile"
value = module.karpenter.instance_profile_arn
}
output "karpenter_instance_profile_id" {
description = "Instance profile's ID"
value = module.karpenter.instance_profile_id
}
output "karpenter_instance_profile_name" {
description = "Name of the instance profile"
value = module.karpenter.instance_profile_name
}
output "karpenter_instance_profile_unique" {
description = "Stable and unique string identifying the IAM instance profile"
value = module.karpenter.instance_profile_unique
}

View File

@@ -6,6 +6,10 @@ terraform {
source = "hashicorp/aws"
version = ">= 3.72"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.10"
}
helm = {
source = "hashicorp/helm"
version = ">= 2.4"
@@ -14,5 +18,9 @@ terraform {
source = "gavinbunney/kubectl"
version = ">= 1.14"
}
null = {
source = "hashicorp/null"
version = ">= 3.0"
}
}
}

View File

@@ -77,8 +77,9 @@ No inputs.
| <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
| <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
| <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready |
| <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
| <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
| <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console |

View File

@@ -10,13 +10,13 @@ provider "kubernetes" {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.22"
cluster_version = "1.24"
region = "eu-west-1"
tags = {

View File

@@ -17,8 +17,13 @@ output "cluster_endpoint" {
value = module.eks.cluster_endpoint
}
output "cluster_name" {
description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_name
}
output "cluster_id" {
description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = module.eks.cluster_id
}

View File

@@ -10,7 +10,7 @@ module "eks_managed_node_group" {
name = "separate-eks-mng"
cluster_name = "my-cluster"
cluster_version = "1.22"
cluster_version = "1.24"
vpc_id = "vpc-1234556abcdef"
subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"]

195
modules/karpenter/README.md Normal file
View File

@@ -0,0 +1,195 @@
# Karpenter Module
Configuration in this directory creates the AWS resources required by Karpenter
## Usage
### All Resources (Default)
In the following example, the Karpenter module will create:
- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize
- An IAM role and instance profile for the nodes created by Karpenter to utilize
- Note: This IAM role ARN will need to be added to the `aws-auth` configmap for nodes to join the cluster successfully
- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc.
This setup is great for running Karpenter on EKS Fargate:
```hcl
module "eks" {
source = "terraform-aws-modules/eks"
# Shown just for connection between cluster and Karpenter sub-module below
manage_aws_auth_configmap = true
aws_auth_roles = [
# We need to add in the Karpenter node IAM role for nodes launched by Karpenter
{
rolearn = module.karpenter.role_arn
username = "system:node:{{EC2PrivateDNSName}}"
groups = [
"system:bootstrappers",
"system:nodes",
]
},
]
...
}
module "karpenter" {
source = "terraform-aws-modules/eks/aws//modules/karpenter"
cluster_name = module.eks.cluster_name
irsa_oidc_provider_arn = module.eks.oidc_provider_arn
irsa_namespace_service_accounts = ["karpenter:karpenter"]
tags = {
Environment = "dev"
Terraform = "true"
}
}
```
### External Node IAM Role (Default)
In the following example, the Karpenter module will create:
- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize
- An IAM instance profile for the nodes created by Karpenter to utilize
- Note: This setup will utilize the existing IAM role created by the EKS Managed Node group which means the role is already populated in the `aws-auth` configmap and no further updates are required.
- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc.
In this scenario, Karpenter would run atop the EKS Managed Node group and scale out nodes as needed from there:
```hcl
module "eks" {
source = "terraform-aws-modules/eks"
# Shown just for connection between cluster and Karpenter sub-module below
eks_managed_node_groups = {
initial = {
instance_types = ["t3.medium"]
min_size = 1
max_size = 3
desired_size = 1
}
}
...
}
module "karpenter" {
source = "terraform-aws-modules/eks/aws//modules/karpenter"
cluster_name = module.eks.cluster_name
irsa_oidc_provider_arn = module.eks.oidc_provider_arn
irsa_namespace_service_accounts = ["karpenter:karpenter"]
create_iam_role = false
iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn
tags = {
Environment = "dev"
Terraform = "true"
}
}
```
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
## Requirements
| Name | Version |
|------|---------|
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
## Providers
| Name | Version |
|------|---------|
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
## Modules
No modules.
## Resources
| Name | Type |
|------|------|
| [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
| [aws_iam_policy.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_role.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
| [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource |
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.irsa_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
## Inputs
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_cluster_ip_family"></a> [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no |
| <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no |
| <a name="input_create"></a> [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no |
| <a name="input_create_iam_role"></a> [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no |
| <a name="input_create_instance_profile"></a> [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no |
| <a name="input_create_irsa"></a> [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no |
| <a name="input_enable_spot_termination"></a> [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no |
| <a name="input_iam_role_additional_policies"></a> [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no |
| <a name="input_iam_role_arn"></a> [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no |
| <a name="input_iam_role_attach_cni_policy"></a> [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no |
| <a name="input_iam_role_description"></a> [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no |
| <a name="input_iam_role_max_session_duration"></a> [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no |
| <a name="input_iam_role_name"></a> [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no |
| <a name="input_iam_role_path"></a> [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `"/"` | no |
| <a name="input_iam_role_permissions_boundary"></a> [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no |
| <a name="input_iam_role_tags"></a> [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no |
| <a name="input_iam_role_use_name_prefix"></a> [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no |
| <a name="input_irsa_assume_role_condition_test"></a> [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no |
| <a name="input_irsa_description"></a> [irsa\_description](#input\_irsa\_description) | IAM role for service accounts description | `string` | `"Karpenter IAM role for service account"` | no |
| <a name="input_irsa_max_session_duration"></a> [irsa\_max\_session\_duration](#input\_irsa\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no |
| <a name="input_irsa_name"></a> [irsa\_name](#input\_irsa\_name) | Name of IAM role for service accounts | `string` | `null` | no |
| <a name="input_irsa_namespace_service_accounts"></a> [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` | <pre>[<br> "karpenter:karpenter"<br>]</pre> | no |
| <a name="input_irsa_oidc_provider_arn"></a> [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no |
| <a name="input_irsa_path"></a> [irsa\_path](#input\_irsa\_path) | Path of IAM role for service accounts | `string` | `"/"` | no |
| <a name="input_irsa_permissions_boundary_arn"></a> [irsa\_permissions\_boundary\_arn](#input\_irsa\_permissions\_boundary\_arn) | Permissions boundary ARN to use for IAM role for service accounts | `string` | `null` | no |
| <a name="input_irsa_ssm_parameter_arns"></a> [irsa\_ssm\_parameter\_arns](#input\_irsa\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` | <pre>[<br> "arn:aws:ssm:*:*:parameter/aws/service/*"<br>]</pre> | no |
| <a name="input_irsa_subnet_account_id"></a> [irsa\_subnet\_account\_id](#input\_irsa\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no |
| <a name="input_irsa_tag_key"></a> [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no |
| <a name="input_irsa_tags"></a> [irsa\_tags](#input\_irsa\_tags) | A map of additional tags to add the the IAM role for service accounts | `map(any)` | `{}` | no |
| <a name="input_irsa_use_name_prefix"></a> [irsa\_use\_name\_prefix](#input\_irsa\_use\_name\_prefix) | Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix | `bool` | `true` | no |
| <a name="input_queue_kms_data_key_reuse_period_seconds"></a> [queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no |
| <a name="input_queue_kms_master_key_id"></a> [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no |
| <a name="input_queue_managed_sse_enabled"></a> [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no |
| <a name="input_queue_name"></a> [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no |
| <a name="input_tags"></a> [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no |
## Outputs
| Name | Description |
|------|-------------|
| <a name="output_event_rules"></a> [event\_rules](#output\_event\_rules) | Map of the event rules created and their attributes |
| <a name="output_instance_profile_arn"></a> [instance\_profile\_arn](#output\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile |
| <a name="output_instance_profile_id"></a> [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID |
| <a name="output_instance_profile_name"></a> [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile |
| <a name="output_instance_profile_unique"></a> [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile |
| <a name="output_irsa_arn"></a> [irsa\_arn](#output\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts |
| <a name="output_irsa_name"></a> [irsa\_name](#output\_irsa\_name) | The name of the IAM role for service accounts |
| <a name="output_irsa_unique_id"></a> [irsa\_unique\_id](#output\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts |
| <a name="output_queue_arn"></a> [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue |
| <a name="output_queue_name"></a> [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue |
| <a name="output_queue_url"></a> [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue |
| <a name="output_role_arn"></a> [role\_arn](#output\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role |
| <a name="output_role_name"></a> [role\_name](#output\_role\_name) | The name of the IAM role |
| <a name="output_role_unique_id"></a> [role\_unique\_id](#output\_role\_unique\_id) | Stable and unique string identifying the IAM role |
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

359
modules/karpenter/main.tf Normal file
View File

@@ -0,0 +1,359 @@
data "aws_partition" "current" {}
data "aws_caller_identity" "current" {}
locals {
account_id = data.aws_caller_identity.current.account_id
partition = data.aws_partition.current.partition
dns_suffix = data.aws_partition.current.dns_suffix
}
################################################################################
# IAM Role for Service Account (IRSA)
# This is used by the Karpenter controller
################################################################################
locals {
create_irsa = var.create && var.create_irsa
irsa_name = coalesce(var.irsa_name, "KarpenterIRSA-${var.cluster_name}")
irsa_oidc_provider_url = replace(var.irsa_oidc_provider_arn, "/^(.*provider/)/", "")
}
data "aws_iam_policy_document" "irsa_assume_role" {
count = local.create_irsa ? 1 : 0
statement {
effect = "Allow"
actions = ["sts:AssumeRoleWithWebIdentity"]
principals {
type = "Federated"
identifiers = [var.irsa_oidc_provider_arn]
}
condition {
test = var.irsa_assume_role_condition_test
variable = "${local.irsa_oidc_provider_url}:sub"
values = [for sa in var.irsa_namespace_service_accounts : "system:serviceaccount:${sa}"]
}
# https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls
condition {
test = var.irsa_assume_role_condition_test
variable = "${local.irsa_oidc_provider_url}:aud"
values = ["sts.amazonaws.com"]
}
}
}
resource "aws_iam_role" "irsa" {
count = local.create_irsa ? 1 : 0
name = var.irsa_use_name_prefix ? null : local.irsa_name
name_prefix = var.irsa_use_name_prefix ? "${local.irsa_name}-" : null
path = var.irsa_path
description = var.irsa_description
assume_role_policy = data.aws_iam_policy_document.irsa_assume_role[0].json
max_session_duration = var.irsa_max_session_duration
permissions_boundary = var.irsa_permissions_boundary_arn
force_detach_policies = true
tags = merge(var.tags, var.irsa_tags)
}
data "aws_iam_policy_document" "irsa" {
count = local.create_irsa ? 1 : 0
statement {
actions = [
"ec2:CreateLaunchTemplate",
"ec2:CreateFleet",
"ec2:CreateTags",
"ec2:DescribeLaunchTemplates",
"ec2:DescribeImages",
"ec2:DescribeInstances",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets",
"ec2:DescribeInstanceTypes",
"ec2:DescribeInstanceTypeOfferings",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeSpotPriceHistory",
"pricing:GetProducts",
]
resources = ["*"]
}
statement {
actions = [
"ec2:TerminateInstances",
"ec2:DeleteLaunchTemplate",
]
resources = ["*"]
condition {
test = "StringEquals"
variable = "ec2:ResourceTag/${var.irsa_tag_key}"
values = [var.cluster_name]
}
}
statement {
actions = ["ec2:RunInstances"]
resources = [
"arn:${local.partition}:ec2:*:${local.account_id}:launch-template/*",
]
condition {
test = "StringEquals"
variable = "ec2:ResourceTag/${var.irsa_tag_key}"
values = [var.cluster_name]
}
}
statement {
actions = ["ec2:RunInstances"]
resources = [
"arn:${local.partition}:ec2:*::image/*",
"arn:${local.partition}:ec2:*:${local.account_id}:instance/*",
"arn:${local.partition}:ec2:*:${local.account_id}:spot-instances-request/*",
"arn:${local.partition}:ec2:*:${local.account_id}:security-group/*",
"arn:${local.partition}:ec2:*:${local.account_id}:volume/*",
"arn:${local.partition}:ec2:*:${local.account_id}:network-interface/*",
"arn:${local.partition}:ec2:*:${coalesce(var.irsa_subnet_account_id, local.account_id)}:subnet/*",
]
}
statement {
actions = ["ssm:GetParameter"]
resources = var.irsa_ssm_parameter_arns
}
statement {
actions = ["iam:PassRole"]
resources = [var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn]
}
dynamic "statement" {
for_each = local.enable_spot_termination ? [1] : []
content {
actions = [
"sqs:DeleteMessage",
"sqs:GetQueueUrl",
"sqs:GetQueueAttributes",
"sqs:ReceiveMessage",
]
resources = [aws_sqs_queue.this[0].arn]
}
}
}
resource "aws_iam_policy" "irsa" {
count = var.create_irsa ? 1 : 0
name_prefix = "${local.irsa_name}-"
path = var.irsa_path
description = var.irsa_description
policy = data.aws_iam_policy_document.irsa[0].json
tags = var.tags
}
resource "aws_iam_role_policy_attachment" "irsa" {
count = local.create_irsa ? 1 : 0
role = aws_iam_role.irsa[0].name
policy_arn = aws_iam_policy.irsa[0].arn
}
################################################################################
# Node Termination Queue
################################################################################
locals {
enable_spot_termination = var.create && var.enable_spot_termination
queue_name = coalesce(var.queue_name, "Karpenter-${var.cluster_name}")
}
resource "aws_sqs_queue" "this" {
count = local.enable_spot_termination ? 1 : 0
name = local.queue_name
message_retention_seconds = 300
sqs_managed_sse_enabled = var.queue_managed_sse_enabled
kms_master_key_id = var.queue_kms_master_key_id
kms_data_key_reuse_period_seconds = var.queue_kms_data_key_reuse_period_seconds
tags = var.tags
}
data "aws_iam_policy_document" "queue" {
count = local.enable_spot_termination ? 1 : 0
statement {
sid = "SqsWrite"
actions = ["sqs:SendMessage"]
resources = [aws_sqs_queue.this[0].arn]
principals {
type = "Service"
identifiers = [
"events.${local.dns_suffix}",
"sqs.${local.dns_suffix}",
]
}
}
}
resource "aws_sqs_queue_policy" "this" {
count = local.enable_spot_termination ? 1 : 0
queue_url = aws_sqs_queue.this[0].url
policy = data.aws_iam_policy_document.queue[0].json
}
################################################################################
# Node Termination Event Rules
################################################################################
locals {
events = {
health_event = {
name = "HealthEvent"
description = "Karpenter interrupt - AWS health event"
event_pattern = {
source = ["aws.health"]
detail-type = ["AWS Health Event"]
}
}
spot_interupt = {
name = "SpotInterrupt"
description = "Karpenter interrupt - EC2 spot instance interruption warning"
event_pattern = {
source = ["aws.ec2"]
detail-type = ["EC2 Spot Instance Interruption Warning"]
}
}
instance_rebalance = {
name = "InstanceRebalance"
description = "Karpenter interrupt - EC2 instance rebalance recommendation"
event_pattern = {
source = ["aws.ec2"]
detail-type = ["EC2 Instance Rebalance Recommendation"]
}
}
instance_state_change = {
name = "InstanceStateChange"
description = "Karpenter interrupt - EC2 instance state-change notification"
event_pattern = {
source = ["aws.ec2"]
detail-type = ["EC2 Instance State-change Notification"]
}
}
}
}
resource "aws_cloudwatch_event_rule" "this" {
for_each = { for k, v in local.events : k => v if local.enable_spot_termination }
name = "Karpenter${each.value.name}-${var.cluster_name}"
description = each.value.description
event_pattern = jsonencode(each.value.event_pattern)
tags = var.tags
}
resource "aws_cloudwatch_event_target" "this" {
for_each = { for k, v in local.events : k => v if local.enable_spot_termination }
rule = aws_cloudwatch_event_rule.this[each.key].name
target_id = "KarpenterInterruptionQueueTarget"
arn = aws_sqs_queue.this[0].arn
}
################################################################################
# Node IAM Role
# This is used by the nodes launched by Karpenter
################################################################################
locals {
create_iam_role = var.create && var.create_iam_role
iam_role_name = coalesce(var.iam_role_name, "Karpenter-${var.cluster_name}")
iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy"
cni_policy = var.cluster_ip_family == "ipv6" ? "${local.iam_role_policy_prefix}/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy"
}
data "aws_iam_policy_document" "assume_role" {
count = local.create_iam_role ? 1 : 0
statement {
sid = "EKSNodeAssumeRole"
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["ec2.${local.dns_suffix}"]
}
}
}
resource "aws_iam_role" "this" {
count = local.create_iam_role ? 1 : 0
name = var.iam_role_use_name_prefix ? null : local.iam_role_name
name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null
path = var.iam_role_path
description = var.iam_role_description
assume_role_policy = data.aws_iam_policy_document.assume_role[0].json
max_session_duration = var.iam_role_max_session_duration
permissions_boundary = var.iam_role_permissions_boundary
force_detach_policies = true
tags = merge(var.tags, var.iam_role_tags)
}
# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group
resource "aws_iam_role_policy_attachment" "this" {
for_each = { for k, v in toset(compact([
"${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy",
"${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly",
var.iam_role_attach_cni_policy ? local.cni_policy : "",
])) : k => v if local.create_iam_role }
policy_arn = each.value
role = aws_iam_role.this[0].name
}
resource "aws_iam_role_policy_attachment" "additional" {
for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role }
policy_arn = each.value
role = aws_iam_role.this[0].name
}
################################################################################
# Node IAM Instance Profile
# This is used by the nodes launched by Karpenter
################################################################################
locals {
external_role_name = try(replace(var.iam_role_arn, "/^(.*role/)/", ""), null)
}
resource "aws_iam_instance_profile" "this" {
count = var.create && var.create_instance_profile ? 1 : 0
name = var.iam_role_use_name_prefix ? null : local.iam_role_name
name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null
path = var.iam_role_path
role = var.create_iam_role ? aws_iam_role.this[0].name : local.external_role_name
tags = merge(var.tags, var.iam_role_tags)
}

View File

@@ -0,0 +1,89 @@
################################################################################
# IAM Role for Service Account (IRSA)
################################################################################
output "irsa_name" {
description = "The name of the IAM role for service accounts"
value = try(aws_iam_role.irsa[0].name, null)
}
output "irsa_arn" {
description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts"
value = try(aws_iam_role.irsa[0].arn, null)
}
output "irsa_unique_id" {
description = "Stable and unique string identifying the IAM role for service accounts"
value = try(aws_iam_role.irsa[0].unique_id, null)
}
################################################################################
# Node Termination Queue
################################################################################
output "queue_arn" {
description = "The ARN of the SQS queue"
value = try(aws_sqs_queue.this[0].arn, null)
}
output "queue_name" {
description = "The name of the created Amazon SQS queue"
value = try(aws_sqs_queue.this[0].name, null)
}
output "queue_url" {
description = "The URL for the created Amazon SQS queue"
value = try(aws_sqs_queue.this[0].url, null)
}
################################################################################
# Node Termination Event Rules
################################################################################
output "event_rules" {
description = "Map of the event rules created and their attributes"
value = aws_cloudwatch_event_rule.this
}
################################################################################
# Node IAM Role
################################################################################
output "role_name" {
description = "The name of the IAM role"
value = try(aws_iam_role.this[0].name, null)
}
output "role_arn" {
description = "The Amazon Resource Name (ARN) specifying the IAM role"
value = try(aws_iam_role.this[0].arn, var.iam_role_arn)
}
output "role_unique_id" {
description = "Stable and unique string identifying the IAM role"
value = try(aws_iam_role.this[0].unique_id, null)
}
################################################################################
# Node IAM Instance Profile
################################################################################
output "instance_profile_arn" {
description = "ARN assigned by AWS to the instance profile"
value = try(aws_iam_instance_profile.this[0].arn, null)
}
output "instance_profile_id" {
description = "Instance profile's ID"
value = try(aws_iam_instance_profile.this[0].id, null)
}
output "instance_profile_name" {
description = "Name of the instance profile"
value = try(aws_iam_instance_profile.this[0].name, null)
}
output "instance_profile_unique" {
description = "Stable and unique string identifying the IAM instance profile"
value = try(aws_iam_instance_profile.this[0].unique_id, null)
}

View File

@@ -0,0 +1,226 @@
variable "create" {
description = "Determines whether to create EKS managed node group or not"
type = bool
default = true
}
variable "tags" {
description = "A map of tags to add to all resources"
type = map(string)
default = {}
}
variable "cluster_name" {
description = "The name of the EKS cluster"
type = string
default = ""
}
################################################################################
# IAM Role for Service Account (IRSA)
################################################################################
variable "create_irsa" {
description = "Determines whether an IAM role for service accounts is created"
type = bool
default = true
}
variable "irsa_name" {
description = "Name of IAM role for service accounts"
type = string
default = null
}
variable "irsa_use_name_prefix" {
description = "Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix"
type = bool
default = true
}
variable "irsa_path" {
description = "Path of IAM role for service accounts"
type = string
default = "/"
}
variable "irsa_description" {
description = "IAM role for service accounts description"
type = string
default = "Karpenter IAM role for service account"
}
variable "irsa_max_session_duration" {
description = "Maximum API session duration in seconds between 3600 and 43200"
type = number
default = null
}
variable "irsa_permissions_boundary_arn" {
description = "Permissions boundary ARN to use for IAM role for service accounts"
type = string
default = null
}
variable "irsa_tags" {
description = "A map of additional tags to add the the IAM role for service accounts"
type = map(any)
default = {}
}
variable "irsa_tag_key" {
description = "Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner"
type = string
default = "karpenter.sh/discovery"
}
variable "irsa_ssm_parameter_arns" {
description = "List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter"
type = list(string)
# https://github.com/aws/karpenter/blob/ed9473a9863ca949b61b9846c8b9f33f35b86dbd/pkg/cloudprovider/aws/ami.go#L105-L123
default = ["arn:aws:ssm:*:*:parameter/aws/service/*"]
}
variable "irsa_subnet_account_id" {
description = "Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account"
type = string
default = ""
}
variable "irsa_oidc_provider_arn" {
description = "OIDC provider arn used in trust policy for IAM role for service accounts"
type = string
default = ""
}
variable "irsa_namespace_service_accounts" {
description = "List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts"
type = list(string)
default = ["karpenter:karpenter"]
}
variable "irsa_assume_role_condition_test" {
description = "Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role"
type = string
default = "StringEquals"
}
################################################################################
# Node Termination Queue
################################################################################
variable "enable_spot_termination" {
description = "Determines whether to enable native spot termination handling"
type = bool
default = true
}
variable "queue_name" {
description = "Name of the SQS queue"
type = string
default = null
}
variable "queue_managed_sse_enabled" {
description = "Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys"
type = bool
default = true
}
variable "queue_kms_master_key_id" {
description = "The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK"
type = string
default = null
}
variable "queue_kms_data_key_reuse_period_seconds" {
description = "The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again"
type = number
default = null
}
################################################################################
# Node IAM Role & Instance Profile
################################################################################
variable "create_iam_role" {
description = "Determines whether an IAM role is created or to use an existing IAM role"
type = bool
default = true
}
variable "cluster_ip_family" {
description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`"
type = string
default = null
}
variable "iam_role_arn" {
description = "Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false`"
type = string
default = null
}
variable "iam_role_name" {
description = "Name to use on IAM role created"
type = string
default = null
}
variable "iam_role_use_name_prefix" {
description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix"
type = bool
default = true
}
variable "iam_role_path" {
description = "IAM role path"
type = string
default = "/"
}
variable "iam_role_description" {
description = "Description of the role"
type = string
default = null
}
variable "iam_role_max_session_duration" {
description = "Maximum API session duration in seconds between 3600 and 43200"
type = number
default = null
}
variable "iam_role_permissions_boundary" {
description = "ARN of the policy that is used to set the permissions boundary for the IAM role"
type = string
default = null
}
variable "iam_role_attach_cni_policy" {
description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster"
type = bool
default = true
}
variable "iam_role_additional_policies" {
description = "Additional policies to be added to the IAM role"
type = list(string)
default = []
}
variable "iam_role_tags" {
description = "A map of additional tags to add to the IAM role created"
type = map(string)
default = {}
}
################################################################################
# Node IAM Instance Profile
################################################################################
variable "create_instance_profile" {
description = "Whether to create an IAM instance profile"
type = bool
default = true
}

View File

@@ -0,0 +1,10 @@
terraform {
required_version = ">= 0.13.1"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 3.72"
}
}
}

View File

@@ -10,7 +10,7 @@ module "self_managed_node_group" {
name = "separate-self-mng"
cluster_name = "my-cluster"
cluster_version = "1.22"
cluster_version = "1.24"
cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com"
cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ=="

View File

@@ -17,8 +17,13 @@ output "cluster_endpoint" {
value = try(aws_eks_cluster.this[0].endpoint, "")
}
output "cluster_name" {
description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = try(aws_eks_cluster.this[0].name, "")
}
output "cluster_id" {
description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready"
value = try(aws_eks_cluster.this[0].id, "")
}

View File

@@ -27,7 +27,7 @@ variable "cluster_name" {
}
variable "cluster_version" {
description = "Kubernetes `<major>.<minor>` version to use for the EKS cluster (i.e.: `1.22`)"
description = "Kubernetes `<major>.<minor>` version to use for the EKS cluster (i.e.: `1.24`)"
type = string
default = null
}