From 7c99bb19cdbf1eb4f4543f9b8e6d29c3a6734a55 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 29 Feb 2024 19:12:50 -0500 Subject: [PATCH] feat: Add support for AL2023 `nodeadm` user data (#2942) --- docs/UPGRADE-20.0.md | 1 + examples/eks_managed_node_group/main.tf | 23 ++ examples/self_managed_node_group/main.tf | 23 ++ examples/user_data/README.md | 14 ++ examples/user_data/main.tf | 198 +++++++++++++++++- examples/user_data/outputs.tf | 43 ++++ .../rendered/al2023/eks-mng-additional.sh | 19 ++ .../rendered/al2023/eks-mng-custom-ami.sh | 41 ++++ .../al2023/eks-mng-custom-template.sh | 44 ++++ .../rendered/al2023/eks-mng-no-op.sh | 0 .../rendered/al2023/self-mng-bootstrap.sh | 40 ++++ .../al2023/self-mng-custom-template.sh | 44 ++++ .../rendered/al2023/self-mng-no-op.sh | 0 .../user_data/templates/al2023_custom.tpl | 14 ++ modules/_user_data/README.md | 3 + modules/_user_data/main.tf | 95 +++++---- modules/_user_data/variables.tf | 22 ++ templates/al2023_user_data.tpl | 13 ++ 18 files changed, 599 insertions(+), 38 deletions(-) create mode 100755 examples/user_data/rendered/al2023/eks-mng-additional.sh create mode 100755 examples/user_data/rendered/al2023/eks-mng-custom-ami.sh create mode 100755 examples/user_data/rendered/al2023/eks-mng-custom-template.sh create mode 100755 examples/user_data/rendered/al2023/eks-mng-no-op.sh create mode 100755 examples/user_data/rendered/al2023/self-mng-bootstrap.sh create mode 100755 examples/user_data/rendered/al2023/self-mng-custom-template.sh create mode 100755 examples/user_data/rendered/al2023/self-mng-no-op.sh create mode 100644 examples/user_data/templates/al2023_custom.tpl create mode 100644 templates/al2023_user_data.tpl diff --git a/docs/UPGRADE-20.0.md b/docs/UPGRADE-20.0.md index 5ae1b35..1f16712 100644 --- a/docs/UPGRADE-20.0.md +++ b/docs/UPGRADE-20.0.md @@ -20,6 +20,7 @@ To give users advanced notice and provide some future direction for this module, 1. The `aws-auth` sub-module will be removed entirely from the project. Since this sub-module is captured in the v20.x releases, users can continue using it even after the module moves forward with the next major version. The long term strategy and direction is cluster access entry and to rely only on the AWS Terraform provider. 2. The default value for `authentication_mode` will change to `API`. Aligning with point 1 above, this is a one way change, but users are free to specify the value of their choosing in place of this default (when the change is made). This module will proceed with an EKS API first strategy. 3. The launch template and autoscaling group usage contained within the EKS managed nodegroup and self-managed nodegroup sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed nodegroup module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed nodegroup sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum. +4. The `platform` variable will be replaced and instead `ami_type` will become the standard across both self-managed nodegroup(s) and EKS managed nodegroup(s). As EKS expands its portfolio of supported operating systems, the `ami_type` is better suited to associate the correct user data format to the respective OS. The `platform` variable is a legacy artifact of self-managed nodegroups but not as descriptive as the `ami_type`, and therefore it will be removed in favor of `ami_type`. ## Additional changes diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 370bb57..17ff2bb 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -86,6 +86,29 @@ module "eks" { } } + # AL2023 node group utilizing new user data format which utilizes nodeadm + # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) + al2023_nodeadm = { + platform = "al2023" + + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + } + ] + } + # Default node group - as provided by AWS EKS using Bottlerocket bottlerocket_default = { # By default, the module creates a launch template to ensure tags are propagated to instances, etc., diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 433e644..a9f228e 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -72,6 +72,29 @@ module "eks" { # Default node group - as provisioned by the module defaults default_node_group = {} + # AL2023 node group utilizing new user data format which utilizes nodeadm + # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh) + al2023_nodeadm = { + platform = "al2023" + + cloudinit_pre_nodeadm = [ + { + content_type = "application/node.eks.aws" + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + } + ] + } + # Bottlerocket node group bottlerocket = { name = "bottlerocket-self-mng" diff --git a/examples/user_data/README.md b/examples/user_data/README.md index 6e1605c..a84b75a 100644 --- a/examples/user_data/README.md +++ b/examples/user_data/README.md @@ -30,6 +30,10 @@ $ terraform apply | Name | Source | Version | |------|--------|---------| +| [eks\_mng\_al2023\_additional](#module\_eks\_mng\_al2023\_additional) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2023\_custom\_ami](#module\_eks\_mng\_al2023\_custom\_ami) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2023\_custom\_template](#module\_eks\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a | +| [eks\_mng\_al2023\_no\_op](#module\_eks\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_additional](#module\_eks\_mng\_al2\_additional) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_custom\_ami](#module\_eks\_mng\_al2\_custom\_ami) | ../../modules/_user_data | n/a | | [eks\_mng\_al2\_custom\_template](#module\_eks\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | @@ -42,6 +46,9 @@ $ terraform apply | [eks\_mng\_windows\_custom\_ami](#module\_eks\_mng\_windows\_custom\_ami) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_custom\_template](#module\_eks\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a | | [eks\_mng\_windows\_no\_op](#module\_eks\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a | +| [self\_mng\_al2023\_bootstrap](#module\_self\_mng\_al2023\_bootstrap) | ../../modules/_user_data | n/a | +| [self\_mng\_al2023\_custom\_template](#module\_self\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a | +| [self\_mng\_al2023\_no\_op](#module\_self\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_bootstrap](#module\_self\_mng\_al2\_bootstrap) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_custom\_template](#module\_self\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a | | [self\_mng\_al2\_no\_op](#module\_self\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a | @@ -56,6 +63,10 @@ $ terraform apply | Name | Type | |------|------| +| [local_file.eks_mng_al2023_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2023_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.eks_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | @@ -68,6 +79,9 @@ $ terraform apply | [local_file.eks_mng_windows_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_windows_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.eks_mng_windows_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2023_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.self_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.self_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | diff --git a/examples/user_data/main.tf b/examples/user_data/main.tf index 5853635..6a90081 100644 --- a/examples/user_data/main.tf +++ b/examples/user_data/main.tf @@ -43,7 +43,6 @@ module "eks_mng_al2_custom_ami" { EOT } - module "eks_mng_al2_custom_template" { source = "../../modules/_user_data" @@ -65,6 +64,107 @@ module "eks_mng_al2_custom_template" { EOT } +################################################################################ +# EKS managed node group - AL2023 +################################################################################ + +module "eks_mng_al2023_no_op" { + source = "../../modules/_user_data" + + platform = "al2023" +} + +module "eks_mng_al2023_additional" { + source = "../../modules/_user_data" + + platform = "al2023" + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] +} + +module "eks_mng_al2023_custom_ami" { + source = "../../modules/_user_data" + + platform = "al2023" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + + enable_bootstrap_user_data = true + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + +module "eks_mng_al2023_custom_template" { + source = "../../modules/_user_data" + + platform = "al2023" + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + enable_bootstrap_user_data = true + user_data_template_path = "${path.module}/templates/al2023_custom.tpl" + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + ################################################################################ # EKS managed node group - Bottlerocket ################################################################################ @@ -80,6 +180,9 @@ module "eks_mng_bottlerocket_additional" { platform = "bottlerocket" + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + bootstrap_extra_args = <<-EOT # extra args added [settings.kernel] @@ -138,6 +241,9 @@ module "eks_mng_windows_additional" { platform = "windows" + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT @@ -245,6 +351,90 @@ module "self_mng_al2_custom_template" { EOT } +################################################################################ +# Self-managed node group - AL2023 +################################################################################ + +module "self_mng_al2023_no_op" { + source = "../../modules/_user_data" + + platform = "al2023" + + is_eks_managed_node_group = false +} + +module "self_mng_al2023_bootstrap" { + source = "../../modules/_user_data" + + platform = "al2023" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + +module "self_mng_al2023_custom_template" { + source = "../../modules/_user_data" + + platform = "al2023" + + enable_bootstrap_user_data = true + is_eks_managed_node_group = false + + cluster_name = local.name + cluster_endpoint = local.cluster_endpoint + cluster_auth_base64 = local.cluster_auth_base64 + + user_data_template_path = "${path.module}/templates/al2023_custom.tpl" + + cloudinit_pre_nodeadm = [{ + content = <<-EOT + --- + apiVersion: node.eks.aws/v1alpha + kind: NodeConfig + spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + EOT + content_type = "application/node.eks.aws" + }] + + cloudinit_post_nodeadm = [{ + content = <<-EOT + echo "All done" + EOT + content_type = "text/x-shellscript; charset=\"us-ascii\"" + }] +} + ################################################################################ # Self-managed node group - Bottlerocket ################################################################################ @@ -269,6 +459,9 @@ module "self_mng_bottlerocket_bootstrap" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + bootstrap_extra_args = <<-EOT # extra args added [settings.kernel] @@ -321,6 +514,9 @@ module "self_mng_windows_bootstrap" { cluster_endpoint = local.cluster_endpoint cluster_auth_base64 = local.cluster_auth_base64 + # Should do nothing + cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr + pre_bootstrap_user_data = <<-EOT [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯' EOT diff --git a/examples/user_data/outputs.tf b/examples/user_data/outputs.tf index 22988e9..b407cce 100644 --- a/examples/user_data/outputs.tf +++ b/examples/user_data/outputs.tf @@ -30,6 +30,30 @@ resource "local_file" "eks_mng_al2_custom_template" { filename = "${path.module}/rendered/al2/eks-mng-custom-template.sh" } +################################################################################ +# EKS managed node group - AL2023 +################################################################################ + +resource "local_file" "eks_mng_al2023_no_op" { + content = base64decode(module.eks_mng_al2023_no_op.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-no-op.sh" +} + +resource "local_file" "eks_mng_al2023_additional" { + content = base64decode(module.eks_mng_al2023_additional.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-additional.sh" +} + +resource "local_file" "eks_mng_al2023_custom_ami" { + content = base64decode(module.eks_mng_al2023_custom_ami.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-custom-ami.sh" +} + +resource "local_file" "eks_mng_al2023_custom_template" { + content = base64decode(module.eks_mng_al2023_custom_template.user_data) + filename = "${path.module}/rendered/al2023/eks-mng-custom-template.sh" +} + ################################################################################ # EKS managed node group - Bottlerocket ################################################################################ @@ -97,6 +121,25 @@ resource "local_file" "self_mng_al2_custom_template" { filename = "${path.module}/rendered/al2/self-mng-custom-template.sh" } +################################################################################ +# Self-managed node group - AL2023 +################################################################################ + +resource "local_file" "self_mng_al2023_no_op" { + content = base64decode(module.self_mng_al2023_no_op.user_data) + filename = "${path.module}/rendered/al2023/self-mng-no-op.sh" +} + +resource "local_file" "self_mng_al2023_bootstrap" { + content = base64decode(module.self_mng_al2023_bootstrap.user_data) + filename = "${path.module}/rendered/al2023/self-mng-bootstrap.sh" +} + +resource "local_file" "self_mng_al2023_custom_template" { + content = base64decode(module.self_mng_al2023_custom_template.user_data) + filename = "${path.module}/rendered/al2023/self-mng-custom-template.sh" +} + ################################################################################ # Self-managed node group - Bottlerocket ################################################################################ diff --git a/examples/user_data/rendered/al2023/eks-mng-additional.sh b/examples/user_data/rendered/al2023/eks-mng-additional.sh new file mode 100755 index 0000000..d8d3a1f --- /dev/null +++ b/examples/user_data/rendered/al2023/eks-mng-additional.sh @@ -0,0 +1,19 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh b/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh new file mode 100755 index 0000000..fe8f07a --- /dev/null +++ b/examples/user_data/rendered/al2023/eks-mng-custom-ami.sh @@ -0,0 +1,41 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + cidr: 172.16.0.0/16 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-custom-template.sh b/examples/user_data/rendered/al2023/eks-mng-custom-template.sh new file mode 100755 index 0000000..6267b85 --- /dev/null +++ b/examples/user_data/rendered/al2023/eks-mng-custom-template.sh @@ -0,0 +1,44 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/eks-mng-no-op.sh b/examples/user_data/rendered/al2023/eks-mng-no-op.sh new file mode 100755 index 0000000..e69de29 diff --git a/examples/user_data/rendered/al2023/self-mng-bootstrap.sh b/examples/user_data/rendered/al2023/self-mng-bootstrap.sh new file mode 100755 index 0000000..9ea0084 --- /dev/null +++ b/examples/user_data/rendered/al2023/self-mng-bootstrap.sh @@ -0,0 +1,40 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/self-mng-custom-template.sh b/examples/user_data/rendered/al2023/self-mng-custom-template.sh new file mode 100755 index 0000000..6267b85 --- /dev/null +++ b/examples/user_data/rendered/al2023/self-mng-custom-template.sh @@ -0,0 +1,44 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha +kind: NodeConfig +spec: + kubelet: + config: + shutdownGracePeriod: 30s + featureGates: + DisableKubeletCloudCredentialProviders: true + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: application/node.eks.aws +Mime-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ex-user-data + apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com + certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + +--MIMEBOUNDARY +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript; charset="us-ascii" +Mime-Version: 1.0 + +echo "All done" + +--MIMEBOUNDARY-- diff --git a/examples/user_data/rendered/al2023/self-mng-no-op.sh b/examples/user_data/rendered/al2023/self-mng-no-op.sh new file mode 100755 index 0000000..e69de29 diff --git a/examples/user_data/templates/al2023_custom.tpl b/examples/user_data/templates/al2023_custom.tpl new file mode 100644 index 0000000..a33aa44 --- /dev/null +++ b/examples/user_data/templates/al2023_custom.tpl @@ -0,0 +1,14 @@ +%{ if enable_bootstrap_user_data ~} +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ${cluster_name} + apiServerEndpoint: ${cluster_endpoint} + certificateAuthority: ${cluster_auth_base64} + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false +%{ endif ~} diff --git a/modules/_user_data/README.md b/modules/_user_data/README.md index 4b50bfe..89edf25 100644 --- a/modules/_user_data/README.md +++ b/modules/_user_data/README.md @@ -26,6 +26,7 @@ No modules. | Name | Type | |------|------| +| [cloudinit_config.al2023_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | | [cloudinit_config.linux_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs @@ -33,6 +34,8 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part | 
list(object({
    content      = string
    content_type = optional(string)
    filename     = optional(string)
    merge_type   = optional(string)
  }))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part | 
list(object({
    content      = string
    content_type = optional(string)
    filename     = optional(string)
    merge_type   = optional(string)
  }))
| `[]` | no | | [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | | [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | diff --git a/modules/_user_data/main.tf b/modules/_user_data/main.tf index 5d9e669..b70279b 100644 --- a/modules/_user_data/main.tf +++ b/modules/_user_data/main.tf @@ -1,65 +1,49 @@ locals { - int_linux_default_user_data = var.create && var.platform == "linux" && (var.enable_bootstrap_user_data || var.user_data_template_path != "") ? base64encode(templatefile( - coalesce(var.user_data_template_path, "${path.module}/../../templates/linux_user_data.tpl"), + template_path = { + al2023 = "${path.module}/../../templates/al2023_user_data.tpl" + bottlerocket = "${path.module}/../../templates/bottlerocket_user_data.tpl" + linux = "${path.module}/../../templates/linux_user_data.tpl" + windows = "${path.module}/../../templates/windows_user_data.tpl" + } + + user_data = base64encode(templatefile( + coalesce(var.user_data_template_path, local.template_path[var.platform]), { # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami enable_bootstrap_user_data = var.enable_bootstrap_user_data + # Required to bootstrap node cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 + # Optional cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr != null ? var.cluster_service_ipv4_cidr : "" bootstrap_extra_args = var.bootstrap_extra_args pre_bootstrap_user_data = var.pre_bootstrap_user_data post_bootstrap_user_data = var.post_bootstrap_user_data } - )) : "" + )) + platform = { + al2023 = { + user_data = var.create ? try(data.cloudinit_config.al2023_eks_managed_node_group[0].rendered, local.user_data) : "" + } bottlerocket = { - user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? base64encode(templatefile( - coalesce(var.user_data_template_path, "${path.module}/../../templates/bottlerocket_user_data.tpl"), - { - # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami - enable_bootstrap_user_data = var.enable_bootstrap_user_data - # Required to bootstrap node - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - # Optional - is appended if using EKS managed node group without custom AMI - # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Bottlerocket pulls this automatically https://github.com/bottlerocket-os/bottlerocket/issues/1866 - bootstrap_extra_args = var.bootstrap_extra_args - } - )) : "" + user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? local.user_data : "" } linux = { - user_data = try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.int_linux_default_user_data) - + user_data = var.create ? try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.user_data) : "" } windows = { - user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? base64encode(templatefile( - coalesce(var.user_data_template_path, "${path.module}/../../templates/windows_user_data.tpl"), - { - # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami - enable_bootstrap_user_data = var.enable_bootstrap_user_data - # Required to bootstrap node - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - # Optional - is appended if using EKS managed node group without custom AMI - # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Not supported yet: https://github.com/awslabs/amazon-eks-ami/issues/805 - bootstrap_extra_args = var.bootstrap_extra_args - pre_bootstrap_user_data = var.pre_bootstrap_user_data - post_bootstrap_user_data = var.post_bootstrap_user_data - } - )) : "" + user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? local.user_data : "" } } } # https://github.com/aws/containers-roadmap/issues/596#issuecomment-675097667 -# An important note is that user data must in MIME multi-part archive format, +# Managed nodegroup data must in MIME multi-part archive format, # as by default, EKS will merge the bootstrapping command required for nodes to join the # cluster with your user data. If you use a custom AMI in your launch template, # this merging will NOT happen and you are responsible for nodes joining the cluster. @@ -74,7 +58,44 @@ data "cloudinit_config" "linux_eks_managed_node_group" { # Prepend to existing user data supplied by AWS EKS part { - content_type = "text/x-shellscript" content = var.pre_bootstrap_user_data + content_type = "text/x-shellscript" + } +} + +# Scenarios: +# +# 1. Do nothing - provide nothing +# 2. Prepend stuff on EKS MNG (before EKS MNG adds its bit at the end) +# 3. Own all of the stuff on self-MNG or EKS MNG w/ custom AMI + +locals { + nodeadm_cloudinit = var.enable_bootstrap_user_data ? concat( + var.cloudinit_pre_nodeadm, + [{ + content_type = "application/node.eks.aws" + content = base64decode(local.user_data) + }], + var.cloudinit_post_nodeadm + ) : var.cloudinit_pre_nodeadm +} + +data "cloudinit_config" "al2023_eks_managed_node_group" { + count = var.create && var.platform == "al2023" && length(local.nodeadm_cloudinit) > 0 ? 1 : 0 + + base64_encode = true + gzip = false + boundary = "MIMEBOUNDARY" + + dynamic "part" { + # Using the index is fine in this context since any change in user data will be a replacement + for_each = { for i, v in local.nodeadm_cloudinit : i => v } + + content { + content = part.value.content + content_type = try(part.value.content_type, null) + filename = try(part.value.filename, null) + merge_type = try(part.value.merge_type, null) + } } } diff --git a/modules/_user_data/variables.tf b/modules/_user_data/variables.tf index 232e1e8..96c1b07 100644 --- a/modules/_user_data/variables.tf +++ b/modules/_user_data/variables.tf @@ -69,3 +69,25 @@ variable "user_data_template_path" { type = string default = "" } + +variable "cloudinit_pre_nodeadm" { + description = "Array of cloud-init document parts that are created before the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +variable "cloudinit_post_nodeadm" { + description = "Array of cloud-init document parts that are created after the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} diff --git a/templates/al2023_user_data.tpl b/templates/al2023_user_data.tpl new file mode 100644 index 0000000..820223f --- /dev/null +++ b/templates/al2023_user_data.tpl @@ -0,0 +1,13 @@ +%{ if enable_bootstrap_user_data ~} +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ${cluster_name} + apiServerEndpoint: ${cluster_endpoint} + certificateAuthority: ${cluster_auth_base64} +%{ if length(cluster_service_ipv4_cidr) > 0 ~} + cidr: ${cluster_service_ipv4_cidr} +%{ endif ~} +%{ endif ~}