feat: Support additional cluster DNS IPs with Bottlerocket based AMIs (#3051)

* Support adding additional cluster dns ranges to bottlerocket template

* Add example for multiple dns ips

* fmt, and tf docs

* fix: Use a list by default for cluster-dns-ip

---------

Co-authored-by: Bryant Biggs <bryantbiggs@gmail.com>

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.89.1
+    rev: v1.90.0
     hooks:
       - id: terraform_fmt
       - id: terraform_docs

examples/user_data/main.tf

@@ -244,6 +244,9 @@ module "eks_mng_bottlerocket_custom_ami" {
   cluster_endpoint     = local.cluster_endpoint
   cluster_auth_base64  = local.cluster_auth_base64
   cluster_service_cidr = local.cluster_service_cidr
+  additional_cluster_dns_ips = [
+    "169.254.20.10"
+  ]
 
   enable_bootstrap_user_data = true
 

examples/user_data/rendered/bottlerocket/eks-mng-custom-ami.toml

@@ -2,7 +2,7 @@
 "cluster-name" = "ex-user-data"
 "api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com"
 "cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ=="
-"cluster-dns-ip" = "192.168.0.10"
+"cluster-dns-ip" = ["192.168.0.10", "169.254.20.10"]
 # extra args added
 [settings.kernel]
 lockdown = "integrity"

examples/user_data/rendered/bottlerocket/self-mng-bootstrap.toml

@@ -2,7 +2,7 @@
 "cluster-name" = "ex-user-data"
 "api-server" = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com"
 "cluster-certificate" = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ=="
-"cluster-dns-ip" = "192.168.0.10"
+"cluster-dns-ip" = ["192.168.0.10"]
 # extra args added
 [settings.kernel]
 lockdown = "integrity"

modules/_user_data/README.md

@@ -36,6 +36,7 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_additional_cluster_dns_ips"></a> [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no |
 | <a name="input_ami_type"></a> [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no |
 | <a name="input_bootstrap_extra_args"></a> [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no |
 | <a name="input_cloudinit_post_nodeadm"></a> [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part | <pre>list(object({<br>    content      = string<br>    content_type = optional(string)<br>    filename     = optional(string)<br>    merge_type   = optional(string)<br>  }))</pre> | `[]` | no |

modules/_user_data/main.tf

@@ -43,6 +43,7 @@ locals {
   }
 
   cluster_service_cidr = try(coalesce(var.cluster_service_ipv4_cidr, var.cluster_service_cidr), "")
+  cluster_dns_ips      = flatten(concat([try(cidrhost(local.cluster_service_cidr, 10), "")], var.additional_cluster_dns_ips))
 
   user_data = base64encode(templatefile(
     coalesce(var.user_data_template_path, local.template_path[local.user_data_type]),
@@ -57,8 +58,9 @@ locals {
 
       cluster_service_cidr = local.cluster_service_cidr
       cluster_ip_family    = var.cluster_ip_family
+
       # Bottlerocket
-      cluster_dns_ip = try(cidrhost(local.cluster_service_cidr, 10), "")
+      cluster_dns_ips = "[${join(", ", formatlist("\"%s\"", local.cluster_dns_ips))}]"
 
       # Optional
       bootstrap_extra_args     = var.bootstrap_extra_args

modules/_user_data/variables.tf

@@ -58,6 +58,12 @@ variable "cluster_ip_family" {
   default     = "ipv4"
 }
 
+variable "additional_cluster_dns_ips" {
+  description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`"
+  type        = list(string)
+  default     = []
+}
+
 # TODO - remove at next breaking change
 variable "cluster_service_ipv4_cidr" {
   description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks"

modules/self-managed-node-group/README.md

@@ -82,6 +82,7 @@ module "self_managed_node_group" {
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_additional_cluster_dns_ips"></a> [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no |
 | <a name="input_ami_id"></a> [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance | `string` | `""` | no |
 | <a name="input_ami_type"></a> [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2_x86_64"` | no |
 | <a name="input_autoscaling_group_tags"></a> [autoscaling\_group\_tags](#input\_autoscaling\_group\_tags) | A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances | `map(string)` | `{}` | no |

modules/self-managed-node-group/main.tf

@@ -65,11 +65,12 @@ module "user_data" {
   ami_type                  = var.ami_type
   is_eks_managed_node_group = false
 
-  cluster_name         = var.cluster_name
+  cluster_name               = var.cluster_name
-  cluster_endpoint     = var.cluster_endpoint
+  cluster_endpoint           = var.cluster_endpoint
-  cluster_auth_base64  = var.cluster_auth_base64
+  cluster_auth_base64        = var.cluster_auth_base64
-  cluster_ip_family    = var.cluster_ip_family
+  cluster_ip_family          = var.cluster_ip_family
-  cluster_service_cidr = var.cluster_service_cidr
+  cluster_service_cidr       = var.cluster_service_cidr
+  additional_cluster_dns_ips = var.additional_cluster_dns_ips
 
   enable_bootstrap_user_data = true
   pre_bootstrap_user_data    = var.pre_bootstrap_user_data

modules/self-managed-node-group/variables.tf

@@ -50,6 +50,12 @@ variable "cluster_ip_family" {
   default     = "ipv4"
 }
 
+variable "additional_cluster_dns_ips" {
+  description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`"
+  type        = list(string)
+  default     = []
+}
+
 variable "pre_bootstrap_user_data" {
   description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`"
   type        = string

templates/bottlerocket_user_data.tpl

@@ -3,6 +3,6 @@
 "cluster-name" = "${cluster_name}"
 "api-server" = "${cluster_endpoint}"
 "cluster-certificate" = "${cluster_auth_base64}"
-"cluster-dns-ip" = "${cluster_dns_ip}"
+"cluster-dns-ip" = ${cluster_dns_ips}
 %{ endif ~}
 ${bootstrap_extra_args ~}