From ebc91bcd37a919a350d872a5b235ccc2a79955a6 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sat, 18 Jun 2022 17:52:30 -0400 Subject: [PATCH] feat: Add support for specifying conrol plane subnets separate from those used by node groups (data plane) (#2113) --- .pre-commit-config.yaml | 4 ++-- README.md | 7 ++++--- docs/compute_resources.md | 6 +++--- docs/irsa_integration.md | 2 +- examples/complete/main.tf | 8 +++++--- examples/eks_managed_node_group/main.tf | 2 +- examples/karpenter/main.tf | 4 ++-- examples/self_managed_node_group/main.tf | 2 +- main.tf | 2 +- modules/eks-managed-node-group/README.md | 2 +- modules/self-managed-node-group/README.md | 2 +- variables.tf | 10 ++++++++-- 12 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0925b0..27d478d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.68.1 + rev: v1.72.1 hooks: - id: terraform_fmt - id: terraform_validate @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_standard_module_structure' - '--args=--only=terraform_workspace_remote' - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.3.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/README.md b/README.md index 8a35630..4677dc0 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ module "eks" { version = "~> 18.0" cluster_name = "my-cluster" - cluster_version = "1.21" + cluster_version = "1.22" cluster_endpoint_private_access = true cluster_endpoint_public_access = true @@ -292,7 +292,8 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`) | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`) | `string` | `null` | no | +| [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled | `bool` | `true` | no | @@ -327,7 +328,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [putin\_khuylo](#input\_putin\_khuylo) | Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo! | `bool` | `true` | no | | [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | | [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Map of self-managed node group definitions to create | `any` | `{}` | no | -| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration | `list(string)` | `[]` | no | +| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets | `list(string)` | `[]` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | | [vpc\_id](#input\_vpc\_id) | ID of the VPC where the cluster and its nodes will be provisioned | `string` | `null` | no | diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 556a2fc..2fefd17 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -141,9 +141,9 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.21" + cluster_version = "1.22" - # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.21 + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.22 self_managed_node_groups = { default = {} } @@ -152,7 +152,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.21" + cluster_version = "1.22" self_managed_node_groups = { bottlerocket = { diff --git a/docs/irsa_integration.md b/docs/irsa_integration.md index 93293e7..6c78bd9 100644 --- a/docs/irsa_integration.md +++ b/docs/irsa_integration.md @@ -8,7 +8,7 @@ module "eks" { source = "terraform-aws-modules/eks/aws" cluster_name = "example" - cluster_version = "1.21" + cluster_version = "1.22" cluster_addons = { vpc-cni = { diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 3702dd1..69014ac 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -13,7 +13,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] @@ -57,8 +57,9 @@ module "eks" { resources = ["secrets"] }] - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + control_plane_subnet_ids = module.vpc.intra_subnets # Extend cluster security group rules cluster_security_group_additional_rules = { @@ -331,6 +332,7 @@ module "vpc" { azs = ["${local.region}a", "${local.region}b", "${local.region}c"] private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + intra_subnets = ["10.0.7.0/28", "10.0.7.16/28", "10.0.7.32/28"] enable_nat_gateway = true single_nat_gateway = true diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 13c43c8..63da4c9 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -7,7 +7,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 4cda4c9..3f43d80 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -77,7 +77,7 @@ provider "helm" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] @@ -92,7 +92,7 @@ provider "kubectl" { load_config_file = false exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index bcb4e7c..116dbc2 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -7,7 +7,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { - api_version = "client.authentication.k8s.io/v1alpha1" + api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] diff --git a/main.tf b/main.tf index 9dad2cc..3254ad1 100644 --- a/main.tf +++ b/main.tf @@ -18,7 +18,7 @@ resource "aws_eks_cluster" "this" { vpc_config { security_group_ids = compact(distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id]))) - subnet_ids = var.subnet_ids + subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) endpoint_private_access = var.cluster_endpoint_private_access endpoint_public_access = var.cluster_endpoint_public_access public_access_cidrs = var.cluster_endpoint_public_access_cidrs diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index d1f4272..179090f 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -10,7 +10,7 @@ module "eks_managed_node_group" { name = "separate-eks-mng" cluster_name = "my-cluster" - cluster_version = "1.21" + cluster_version = "1.22" vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 36bc98e..3cba488 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -10,7 +10,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = "my-cluster" - cluster_version = "1.21" + cluster_version = "1.22" cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" diff --git a/variables.tf b/variables.tf index 2890834..094711f 100644 --- a/variables.tf +++ b/variables.tf @@ -27,7 +27,7 @@ variable "cluster_name" { } variable "cluster_version" { - description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.21`)" + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`)" type = string default = null } @@ -44,8 +44,14 @@ variable "cluster_additional_security_group_ids" { default = [] } +variable "control_plane_subnet_ids" { + description = "A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane" + type = list(string) + default = [] +} + variable "subnet_ids" { - description = "A list of subnet IDs where the EKS cluster (ENIs) will be provisioned along with the nodes/node groups. Node groups can be deployed within a different set of subnet IDs from within the node group configuration" + description = "A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets" type = list(string) default = [] }