feat: Add support for AL2023 nodeadm user data (#2942)

docs/UPGRADE-20.0.md

@@ -20,6 +20,7 @@ To give users advanced notice and provide some future direction for this module,
 1. The `aws-auth` sub-module will be removed entirely from the project. Since this sub-module is captured in the v20.x releases, users can continue using it even after the module moves forward with the next major version. The long term strategy and direction is cluster access entry and to rely only on the AWS Terraform provider.
 2. The default value for `authentication_mode` will change to `API`. Aligning with point 1 above, this is a one way change, but users are free to specify the value of their choosing in place of this default (when the change is made). This module will proceed with an EKS API first strategy.
 3. The launch template and autoscaling group usage contained within the EKS managed nodegroup and self-managed nodegroup sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed nodegroup module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed nodegroup sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum.
+4. The `platform` variable will be replaced and instead `ami_type` will become the standard across both self-managed nodegroup(s) and EKS managed nodegroup(s). As EKS expands its portfolio of supported operating systems, the `ami_type` is better suited to associate the correct user data format to the respective OS. The `platform` variable is a legacy artifact of self-managed nodegroups but not as descriptive as the `ami_type`, and therefore it will be removed in favor of `ami_type`.
 
 ## Additional changes
 

examples/eks_managed_node_group/main.tf

@@ -86,6 +86,29 @@ module "eks" {
       }
     }
 
+    # AL2023 node group utilizing new user data format which utilizes nodeadm
+    # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh)
+    al2023_nodeadm = {
+      platform = "al2023"
+
+      cloudinit_pre_nodeadm = [
+        {
+          content_type = "application/node.eks.aws"
+          content      = <<-EOT
+            ---
+            apiVersion: node.eks.aws/v1alpha
+            kind: NodeConfig
+            spec:
+              kubelet:
+                config:
+                  shutdownGracePeriod: 30s
+                  featureGates:
+                    DisableKubeletCloudCredentialProviders: true
+          EOT
+        }
+      ]
+    }
+
     # Default node group - as provided by AWS EKS using Bottlerocket
     bottlerocket_default = {
       # By default, the module creates a launch template to ensure tags are propagated to instances, etc.,

examples/self_managed_node_group/main.tf

@@ -72,6 +72,29 @@ module "eks" {
     # Default node group - as provisioned by the module defaults
     default_node_group = {}
 
+    # AL2023 node group utilizing new user data format which utilizes nodeadm
+    # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh)
+    al2023_nodeadm = {
+      platform = "al2023"
+
+      cloudinit_pre_nodeadm = [
+        {
+          content_type = "application/node.eks.aws"
+          content      = <<-EOT
+            ---
+            apiVersion: node.eks.aws/v1alpha
+            kind: NodeConfig
+            spec:
+              kubelet:
+                config:
+                  shutdownGracePeriod: 30s
+                  featureGates:
+                    DisableKubeletCloudCredentialProviders: true
+          EOT
+        }
+      ]
+    }
+
     # Bottlerocket node group
     bottlerocket = {
       name = "bottlerocket-self-mng"

examples/user_data/README.md

@@ -30,6 +30,10 @@ $ terraform apply
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_eks_mng_al2023_additional"></a> [eks\_mng\_al2023\_additional](#module\_eks\_mng\_al2023\_additional) | ../../modules/_user_data | n/a |
+| <a name="module_eks_mng_al2023_custom_ami"></a> [eks\_mng\_al2023\_custom\_ami](#module\_eks\_mng\_al2023\_custom\_ami) | ../../modules/_user_data | n/a |
+| <a name="module_eks_mng_al2023_custom_template"></a> [eks\_mng\_al2023\_custom\_template](#module\_eks\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a |
+| <a name="module_eks_mng_al2023_no_op"></a> [eks\_mng\_al2023\_no\_op](#module\_eks\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_al2_additional"></a> [eks\_mng\_al2\_additional](#module\_eks\_mng\_al2\_additional) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_al2_custom_ami"></a> [eks\_mng\_al2\_custom\_ami](#module\_eks\_mng\_al2\_custom\_ami) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_al2_custom_template"></a> [eks\_mng\_al2\_custom\_template](#module\_eks\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a |
@@ -42,6 +46,9 @@ $ terraform apply
 | <a name="module_eks_mng_windows_custom_ami"></a> [eks\_mng\_windows\_custom\_ami](#module\_eks\_mng\_windows\_custom\_ami) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_windows_custom_template"></a> [eks\_mng\_windows\_custom\_template](#module\_eks\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_windows_no_op"></a> [eks\_mng\_windows\_no\_op](#module\_eks\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a |
+| <a name="module_self_mng_al2023_bootstrap"></a> [self\_mng\_al2023\_bootstrap](#module\_self\_mng\_al2023\_bootstrap) | ../../modules/_user_data | n/a |
+| <a name="module_self_mng_al2023_custom_template"></a> [self\_mng\_al2023\_custom\_template](#module\_self\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a |
+| <a name="module_self_mng_al2023_no_op"></a> [self\_mng\_al2023\_no\_op](#module\_self\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a |
 | <a name="module_self_mng_al2_bootstrap"></a> [self\_mng\_al2\_bootstrap](#module\_self\_mng\_al2\_bootstrap) | ../../modules/_user_data | n/a |
 | <a name="module_self_mng_al2_custom_template"></a> [self\_mng\_al2\_custom\_template](#module\_self\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a |
 | <a name="module_self_mng_al2_no_op"></a> [self\_mng\_al2\_no\_op](#module\_self\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a |
@@ -56,6 +63,10 @@ $ terraform apply
 
 | Name | Type |
 |------|------|
+| [local_file.eks_mng_al2023_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.eks_mng_al2023_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.eks_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.eks_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_al2_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_al2_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
@@ -68,6 +79,9 @@ $ terraform apply
 | [local_file.eks_mng_windows_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_windows_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_windows_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.self_mng_al2023_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.self_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.self_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.self_mng_al2_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.self_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.self_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |

examples/user_data/main.tf

@@ -43,7 +43,6 @@ module "eks_mng_al2_custom_ami" {
   EOT
 }
 
-
 module "eks_mng_al2_custom_template" {
   source = "../../modules/_user_data"
 
@@ -65,6 +64,107 @@ module "eks_mng_al2_custom_template" {
   EOT
 }
 
+################################################################################
+# EKS managed node group - AL2023
+################################################################################
+
+module "eks_mng_al2023_no_op" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+}
+
+module "eks_mng_al2023_additional" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+}
+
+module "eks_mng_al2023_custom_ami" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  cluster_name              = local.name
+  cluster_endpoint          = local.cluster_endpoint
+  cluster_auth_base64       = local.cluster_auth_base64
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
+  enable_bootstrap_user_data = true
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
+module "eks_mng_al2023_custom_template" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  cluster_name        = local.name
+  cluster_endpoint    = local.cluster_endpoint
+  cluster_auth_base64 = local.cluster_auth_base64
+
+  enable_bootstrap_user_data = true
+  user_data_template_path    = "${path.module}/templates/al2023_custom.tpl"
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
 ################################################################################
 # EKS managed node group - Bottlerocket
 ################################################################################
@@ -80,6 +180,9 @@ module "eks_mng_bottlerocket_additional" {
 
   platform = "bottlerocket"
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   bootstrap_extra_args = <<-EOT
     # extra args added
     [settings.kernel]
@@ -138,6 +241,9 @@ module "eks_mng_windows_additional" {
 
   platform = "windows"
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   pre_bootstrap_user_data = <<-EOT
     [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯'
   EOT
@@ -245,6 +351,90 @@ module "self_mng_al2_custom_template" {
   EOT
 }
 
+################################################################################
+# Self-managed node group - AL2023
+################################################################################
+
+module "self_mng_al2023_no_op" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  is_eks_managed_node_group = false
+}
+
+module "self_mng_al2023_bootstrap" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  enable_bootstrap_user_data = true
+  is_eks_managed_node_group  = false
+
+  cluster_name        = local.name
+  cluster_endpoint    = local.cluster_endpoint
+  cluster_auth_base64 = local.cluster_auth_base64
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
+module "self_mng_al2023_custom_template" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  enable_bootstrap_user_data = true
+  is_eks_managed_node_group  = false
+
+  cluster_name        = local.name
+  cluster_endpoint    = local.cluster_endpoint
+  cluster_auth_base64 = local.cluster_auth_base64
+
+  user_data_template_path = "${path.module}/templates/al2023_custom.tpl"
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
 ################################################################################
 # Self-managed node group - Bottlerocket
 ################################################################################
@@ -269,6 +459,9 @@ module "self_mng_bottlerocket_bootstrap" {
   cluster_endpoint    = local.cluster_endpoint
   cluster_auth_base64 = local.cluster_auth_base64
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   bootstrap_extra_args = <<-EOT
     # extra args added
     [settings.kernel]
@@ -321,6 +514,9 @@ module "self_mng_windows_bootstrap" {
   cluster_endpoint    = local.cluster_endpoint
   cluster_auth_base64 = local.cluster_auth_base64
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   pre_bootstrap_user_data = <<-EOT
     [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯'
   EOT

examples/user_data/outputs.tf

@@ -30,6 +30,30 @@ resource "local_file" "eks_mng_al2_custom_template" {
   filename = "${path.module}/rendered/al2/eks-mng-custom-template.sh"
 }
 
+################################################################################
+# EKS managed node group - AL2023
+################################################################################
+
+resource "local_file" "eks_mng_al2023_no_op" {
+  content  = base64decode(module.eks_mng_al2023_no_op.user_data)
+  filename = "${path.module}/rendered/al2023/eks-mng-no-op.sh"
+}
+
+resource "local_file" "eks_mng_al2023_additional" {
+  content  = base64decode(module.eks_mng_al2023_additional.user_data)
+  filename = "${path.module}/rendered/al2023/eks-mng-additional.sh"
+}
+
+resource "local_file" "eks_mng_al2023_custom_ami" {
+  content  = base64decode(module.eks_mng_al2023_custom_ami.user_data)
+  filename = "${path.module}/rendered/al2023/eks-mng-custom-ami.sh"
+}
+
+resource "local_file" "eks_mng_al2023_custom_template" {
+  content  = base64decode(module.eks_mng_al2023_custom_template.user_data)
+  filename = "${path.module}/rendered/al2023/eks-mng-custom-template.sh"
+}
+
 ################################################################################
 # EKS managed node group - Bottlerocket
 ################################################################################
@@ -97,6 +121,25 @@ resource "local_file" "self_mng_al2_custom_template" {
   filename = "${path.module}/rendered/al2/self-mng-custom-template.sh"
 }
 
+################################################################################
+# Self-managed node group - AL2023
+################################################################################
+
+resource "local_file" "self_mng_al2023_no_op" {
+  content  = base64decode(module.self_mng_al2023_no_op.user_data)
+  filename = "${path.module}/rendered/al2023/self-mng-no-op.sh"
+}
+
+resource "local_file" "self_mng_al2023_bootstrap" {
+  content  = base64decode(module.self_mng_al2023_bootstrap.user_data)
+  filename = "${path.module}/rendered/al2023/self-mng-bootstrap.sh"
+}
+
+resource "local_file" "self_mng_al2023_custom_template" {
+  content  = base64decode(module.self_mng_al2023_custom_template.user_data)
+  filename = "${path.module}/rendered/al2023/self-mng-custom-template.sh"
+}
+
 ################################################################################
 # Self-managed node group - Bottlerocket
 ################################################################################

examples/user_data/rendered/al2023/eks-mng-additional.sh

@@ -0,0 +1,19 @@
+Content-Type: multipart/mixed; boundary="MIMEBOUNDARY"
+MIME-Version: 1.0
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha
+kind: NodeConfig
+spec:
+  kubelet:
+    config:
+      shutdownGracePeriod: 30s
+      featureGates:
+        DisableKubeletCloudCredentialProviders: true
+
+--MIMEBOUNDARY--

examples/user_data/rendered/al2023/eks-mng-custom-ami.sh

@@ -0,0 +1,41 @@
+Content-Type: multipart/mixed; boundary="MIMEBOUNDARY"
+MIME-Version: 1.0
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha
+kind: NodeConfig
+spec:
+  kubelet:
+    config:
+      shutdownGracePeriod: 30s
+      featureGates:
+        DisableKubeletCloudCredentialProviders: true
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ex-user-data
+    apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com
+    certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
+    cidr: 172.16.0.0/16
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: text/x-shellscript; charset="us-ascii"
+Mime-Version: 1.0
+
+echo "All done"
+
+--MIMEBOUNDARY--

examples/user_data/rendered/al2023/eks-mng-custom-template.sh

@@ -0,0 +1,44 @@
+Content-Type: multipart/mixed; boundary="MIMEBOUNDARY"
+MIME-Version: 1.0
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha
+kind: NodeConfig
+spec:
+  kubelet:
+    config:
+      shutdownGracePeriod: 30s
+      featureGates:
+        DisableKubeletCloudCredentialProviders: true
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ex-user-data
+    apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com
+    certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
+  containerd:
+    config: |
+      [plugins."io.containerd.grpc.v1.cri".containerd]
+      discard_unpacked_layers = false
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: text/x-shellscript; charset="us-ascii"
+Mime-Version: 1.0
+
+echo "All done"
+
+--MIMEBOUNDARY--

examples/user_data/rendered/al2023/self-mng-bootstrap.sh

@@ -0,0 +1,40 @@
+Content-Type: multipart/mixed; boundary="MIMEBOUNDARY"
+MIME-Version: 1.0
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha
+kind: NodeConfig
+spec:
+  kubelet:
+    config:
+      shutdownGracePeriod: 30s
+      featureGates:
+        DisableKubeletCloudCredentialProviders: true
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ex-user-data
+    apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com
+    certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: text/x-shellscript; charset="us-ascii"
+Mime-Version: 1.0
+
+echo "All done"
+
+--MIMEBOUNDARY--

examples/user_data/rendered/al2023/self-mng-custom-template.sh

@@ -0,0 +1,44 @@
+Content-Type: multipart/mixed; boundary="MIMEBOUNDARY"
+MIME-Version: 1.0
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha
+kind: NodeConfig
+spec:
+  kubelet:
+    config:
+      shutdownGracePeriod: 30s
+      featureGates:
+        DisableKubeletCloudCredentialProviders: true
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: application/node.eks.aws
+Mime-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ex-user-data
+    apiServerEndpoint: https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com
+    certificateAuthority: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
+  containerd:
+    config: |
+      [plugins."io.containerd.grpc.v1.cri".containerd]
+      discard_unpacked_layers = false
+
+--MIMEBOUNDARY
+Content-Transfer-Encoding: 7bit
+Content-Type: text/x-shellscript; charset="us-ascii"
+Mime-Version: 1.0
+
+echo "All done"
+
+--MIMEBOUNDARY--

examples/user_data/templates/al2023_custom.tpl

@@ -0,0 +1,14 @@
+%{ if enable_bootstrap_user_data ~}
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ${cluster_name}
+    apiServerEndpoint: ${cluster_endpoint}
+    certificateAuthority: ${cluster_auth_base64}
+  containerd:
+    config: |
+      [plugins."io.containerd.grpc.v1.cri".containerd]
+      discard_unpacked_layers = false
+%{ endif ~}

modules/_user_data/README.md

@@ -26,6 +26,7 @@ No modules.
 
 | Name | Type |
 |------|------|
+| [cloudinit_config.al2023_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source |
 | [cloudinit_config.linux_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source |
 
 ## Inputs
@@ -33,6 +34,8 @@ No modules.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_bootstrap_extra_args"></a> [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `platform` = `bottlerocket`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no |
+| <a name="input_cloudinit_post_nodeadm"></a> [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part | <pre>list(object({<br>    content      = string<br>    content_type = optional(string)<br>    filename     = optional(string)<br>    merge_type   = optional(string)<br>  }))</pre> | `[]` | no |
+| <a name="input_cloudinit_pre_nodeadm"></a> [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part | <pre>list(object({<br>    content      = string<br>    content_type = optional(string)<br>    filename     = optional(string)<br>    merge_type   = optional(string)<br>  }))</pre> | `[]` | no |
 | <a name="input_cluster_auth_base64"></a> [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no |
 | <a name="input_cluster_endpoint"></a> [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no |

modules/_user_data/main.tf

@@ -1,65 +1,49 @@
 
 locals {
-  int_linux_default_user_data = var.create && var.platform == "linux" && (var.enable_bootstrap_user_data || var.user_data_template_path != "") ? base64encode(templatefile(
-    coalesce(var.user_data_template_path, "${path.module}/../../templates/linux_user_data.tpl"),
+  template_path = {
+    al2023       = "${path.module}/../../templates/al2023_user_data.tpl"
+    bottlerocket = "${path.module}/../../templates/bottlerocket_user_data.tpl"
+    linux        = "${path.module}/../../templates/linux_user_data.tpl"
+    windows      = "${path.module}/../../templates/windows_user_data.tpl"
+  }
+
+  user_data = base64encode(templatefile(
+    coalesce(var.user_data_template_path, local.template_path[var.platform]),
     {
       # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami
       enable_bootstrap_user_data = var.enable_bootstrap_user_data
+
       # Required to bootstrap node
       cluster_name        = var.cluster_name
       cluster_endpoint    = var.cluster_endpoint
       cluster_auth_base64 = var.cluster_auth_base64
+
       # Optional
       cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr != null ? var.cluster_service_ipv4_cidr : ""
       bootstrap_extra_args      = var.bootstrap_extra_args
       pre_bootstrap_user_data   = var.pre_bootstrap_user_data
       post_bootstrap_user_data  = var.post_bootstrap_user_data
     }
-  )) : ""
+  ))
+
   platform = {
+    al2023 = {
+      user_data = var.create ? try(data.cloudinit_config.al2023_eks_managed_node_group[0].rendered, local.user_data) : ""
+    }
     bottlerocket = {
-      user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? base64encode(templatefile(
-        coalesce(var.user_data_template_path, "${path.module}/../../templates/bottlerocket_user_data.tpl"),
-        {
-          # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami
-          enable_bootstrap_user_data = var.enable_bootstrap_user_data
-          # Required to bootstrap node
-          cluster_name        = var.cluster_name
-          cluster_endpoint    = var.cluster_endpoint
-          cluster_auth_base64 = var.cluster_auth_base64
-          # Optional - is appended if using EKS managed node group without custom AMI
-          # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Bottlerocket pulls this automatically https://github.com/bottlerocket-os/bottlerocket/issues/1866
-          bootstrap_extra_args = var.bootstrap_extra_args
-        }
-      )) : ""
+      user_data = var.create && var.platform == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? local.user_data : ""
     }
     linux = {
-      user_data = try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.int_linux_default_user_data)
-
+      user_data = var.create ? try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.user_data) : ""
     }
     windows = {
-      user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? base64encode(templatefile(
-        coalesce(var.user_data_template_path, "${path.module}/../../templates/windows_user_data.tpl"),
-        {
-          # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami
-          enable_bootstrap_user_data = var.enable_bootstrap_user_data
-          # Required to bootstrap node
-          cluster_name        = var.cluster_name
-          cluster_endpoint    = var.cluster_endpoint
-          cluster_auth_base64 = var.cluster_auth_base64
-          # Optional - is appended if using EKS managed node group without custom AMI
-          # cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr # Not supported yet: https://github.com/awslabs/amazon-eks-ami/issues/805
-          bootstrap_extra_args     = var.bootstrap_extra_args
-          pre_bootstrap_user_data  = var.pre_bootstrap_user_data
-          post_bootstrap_user_data = var.post_bootstrap_user_data
-        }
-      )) : ""
+      user_data = var.create && var.platform == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? local.user_data : ""
     }
   }
 }
 
 # https://github.com/aws/containers-roadmap/issues/596#issuecomment-675097667
-# An important note is that user data must in MIME multi-part archive format,
+# Managed nodegroup data must in MIME multi-part archive format,
 # as by default, EKS will merge the bootstrapping command required for nodes to join the
 # cluster with your user data. If you use a custom AMI in your launch template,
 # this merging will NOT happen and you are responsible for nodes joining the cluster.
@@ -74,7 +58,44 @@ data "cloudinit_config" "linux_eks_managed_node_group" {
 
   # Prepend to existing user data supplied by AWS EKS
   part {
-    content_type = "text/x-shellscript"
     content      = var.pre_bootstrap_user_data
+    content_type = "text/x-shellscript"
+  }
+}
+
+# Scenarios:
+#
+# 1. Do nothing - provide nothing
+# 2. Prepend stuff on EKS MNG (before EKS MNG adds its bit at the end)
+# 3. Own all of the stuff on self-MNG or EKS MNG w/ custom AMI
+
+locals {
+  nodeadm_cloudinit = var.enable_bootstrap_user_data ? concat(
+    var.cloudinit_pre_nodeadm,
+    [{
+      content_type = "application/node.eks.aws"
+      content      = base64decode(local.user_data)
+    }],
+    var.cloudinit_post_nodeadm
+  ) : var.cloudinit_pre_nodeadm
+}
+
+data "cloudinit_config" "al2023_eks_managed_node_group" {
+  count = var.create && var.platform == "al2023" && length(local.nodeadm_cloudinit) > 0 ? 1 : 0
+
+  base64_encode = true
+  gzip          = false
+  boundary      = "MIMEBOUNDARY"
+
+  dynamic "part" {
+    # Using the index is fine in this context since any change in user data will be a replacement
+    for_each = { for i, v in local.nodeadm_cloudinit : i => v }
+
+    content {
+      content      = part.value.content
+      content_type = try(part.value.content_type, null)
+      filename     = try(part.value.filename, null)
+      merge_type   = try(part.value.merge_type, null)
+    }
   }
 }

modules/_user_data/variables.tf

@@ -69,3 +69,25 @@ variable "user_data_template_path" {
   type        = string
   default     = ""
 }
+
+variable "cloudinit_pre_nodeadm" {
+  description = "Array of cloud-init document parts that are created before the nodeadm document part"
+  type = list(object({
+    content      = string
+    content_type = optional(string)
+    filename     = optional(string)
+    merge_type   = optional(string)
+  }))
+  default = []
+}
+
+variable "cloudinit_post_nodeadm" {
+  description = "Array of cloud-init document parts that are created after the nodeadm document part"
+  type = list(object({
+    content      = string
+    content_type = optional(string)
+    filename     = optional(string)
+    merge_type   = optional(string)
+  }))
+  default = []
+}

templates/al2023_user_data.tpl

@@ -0,0 +1,13 @@
+%{ if enable_bootstrap_user_data ~}
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ${cluster_name}
+    apiServerEndpoint: ${cluster_endpoint}
+    certificateAuthority: ${cluster_auth_base64}
+%{ if length(cluster_service_ipv4_cidr) > 0 ~}
+    cidr: ${cluster_service_ipv4_cidr}
+%{ endif ~}
+%{ endif ~}