diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 3de86a5..dad516e 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -12,6 +12,33 @@ Configuration in this directory creates an AWS EKS cluster with various EKS Mana See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details. +## Container Runtime & User Data + +When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootrap.sh script invocation and therefore its configuration flags (this is provide by the EKS Managed Node Group service in the node user data), a work around for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844). + +```hcl + ... + # Demo of containerd usage when not specifying a custom AMI ID + # (merged into user data before EKS MNG provided user data) + containerd = { + name = "containerd" + + # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + ... +``` + ## Usage To run this example you need to execute: @@ -63,6 +90,9 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default_arm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index f95f5d6..2739f2a 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -133,7 +133,7 @@ module "eks" { # Custom AMI, using module provided bootstrap data bottlerocket_custom = { # Current bottlerocket AMI - ami_id = "ami-0ff61e0bcfc81dc94" + ami_id = data.aws_ami.eks_default_bottlerocket.image_id platform = "bottlerocket" # use module user data template to boostrap @@ -165,7 +165,7 @@ module "eks" { custom_ami = { ami_type = "AL2_ARM_64" # Current default AMI used by managed node groups - pseudo "custom" - ami_id = "ami-01dc0aa438e3214c2" # ARM + ami_id = data.aws_ami.eks_default_arm.image_id # This will ensure the boostrap user data is used to join the node # By default, EKS managed node groups will not append bootstrap script; @@ -176,6 +176,25 @@ module "eks" { instance_types = ["t4g.medium"] } + # Demo of containerd usage when not specifying a custom AMI ID + # (merged into user data before EKS MNG provided user data) + containerd = { + name = "containerd" + + # See issue https://github.com/awslabs/amazon-eks-ami/issues/844 + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + export KUBELET_EXTRA_ARGS="--max-pods=110" + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + # Complete complete = { name = "complete-eks-mng" @@ -187,23 +206,23 @@ module "eks" { max_size = 7 desired_size = 1 - ami_id = "ami-0caf35bc73450c396" + ami_id = data.aws_ami.eks_default.image_id enable_bootstrap_user_data = true bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" + echo "you are free little kubelet!" EOT capacity_type = "SPOT" disk_size = 256 force_update_version = true - instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large", "m3.large", "m4.large"] + instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"] labels = { GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" @@ -619,3 +638,33 @@ resource "aws_iam_policy" "node_additional" { tags = local.tags } + +data "aws_ami" "eks_default" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amazon-eks-node-${local.cluster_version}-v*"] + } +} + +data "aws_ami" "eks_default_arm" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amazon-eks-arm64-node-${local.cluster_version}-v*"] + } +} + +data "aws_ami" "eks_default_bottlerocket" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"] + } +} diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index 3dfc50d..8356943 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -55,7 +55,8 @@ Note that this example may create resources which cost money. Run `terraform des | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | -| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 90d97d5..b8598d7 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -94,7 +94,7 @@ module "eks" { name = "bottlerocket-self-mng" platform = "bottlerocket" - ami_id = data.aws_ami.bottlerocket_ami.id + ami_id = data.aws_ami.eks_default_bottlerocket.id instance_type = "m5.large" desired_size = 2 key_name = aws_key_pair.this.key_name @@ -159,16 +159,16 @@ module "eks" { max_size = 7 desired_size = 1 - ami_id = "ami-0caf35bc73450c396" + ami_id = data.aws_ami.eks_default.id bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'" pre_bootstrap_user_data = <<-EOT - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false EOT post_bootstrap_user_data = <<-EOT - echo "you are free little kubelet!" + echo "you are free little kubelet!" EOT disk_size = 256 @@ -374,7 +374,17 @@ resource "aws_kms_key" "eks" { tags = local.tags } -data "aws_ami" "bottlerocket_ami" { +data "aws_ami" "eks_default" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amazon-eks-node-${local.cluster_version}-v*"] + } +} + +data "aws_ami" "eks_default_bottlerocket" { most_recent = true owners = ["amazon"] diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index bbe0210..71972b2 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -30,8 +30,14 @@ module "user_data" { ################################################################################ locals { - use_custom_launch_template = var.launch_template_name != "" - launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group") + # There are 4 scenarios here that have to be considered for `use_custom_launch_template`: + # 1. `var.create_launch_template = false && var.launch_template_name == ""` => EKS MNG will use its own default LT + # 2. `var.create_launch_template = false && var.launch_template_name == "something"` => User provided custom LT will be used + # 3. `var.create_launch_template = true && var.launch_template_name == ""` => Custom LT will be used, module will provide a default name + # 4. `var.create_launch_template = true && var.launch_template_name == "something"` => Custom LT will be used, LT name is provided by user + use_custom_launch_template = var.create_launch_template || var.launch_template_name != "" + + launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group") } resource "aws_launch_template" "this" {