From 0d77e30075e20ff16fa7357987a240e93f5f28ce Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Sat, 7 Nov 2020 23:03:12 +0100 Subject: [PATCH] feat: Add EKS Fargate support (#1067) Co-authored-by: Simon Gurcke Co-authored-by: Daniel Piddock <33028589+dpiddockcmp@users.noreply.github.com> --- README.md | 7 +++ aws_auth.tf | 6 +- examples/fargate/main.tf | 112 ++++++++++++++++++++++++++++++++++ examples/fargate/outputs.tf | 29 +++++++++ examples/fargate/variables.tf | 52 ++++++++++++++++ fargate.tf | 22 +++++++ modules/fargate/README.md | 54 ++++++++++++++++ modules/fargate/data.tf | 17 ++++++ modules/fargate/fargate.tf | 29 +++++++++ modules/fargate/locals.tf | 10 +++ modules/fargate/outputs.tf | 27 ++++++++ modules/fargate/variables.tf | 59 ++++++++++++++++++ outputs.tf | 20 ++++++ variables.tf | 18 ++++++ 14 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 examples/fargate/main.tf create mode 100644 examples/fargate/outputs.tf create mode 100644 examples/fargate/variables.tf create mode 100644 fargate.tf create mode 100644 modules/fargate/README.md create mode 100644 modules/fargate/data.tf create mode 100644 modules/fargate/fargate.tf create mode 100644 modules/fargate/locals.tf create mode 100644 modules/fargate/outputs.tf create mode 100644 modules/fargate/variables.tf diff --git a/README.md b/README.md index d4ae770..1784cca 100644 --- a/README.md +++ b/README.md @@ -187,8 +187,11 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | | config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no | | create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| create\_fargate\_pod\_execution\_role | Controls if the EKS Fargate pod execution IAM role should be created. | `bool` | `true` | no | | eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no | | enable\_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no | +| fargate\_pod\_execution\_role\_name | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | +| fargate\_profiles | Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details | `any` | `{}` | no | | iam\_path | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | | kubeconfig\_aws\_authenticator\_additional\_args | Any additional arguments to pass to the authenticator such as the role to assume. e.g. ["-r", "MyEksRole"]. | `list(string)` | `[]` | no | | kubeconfig\_aws\_authenticator\_command | Command to use to fetch AWS EKS credentials. | `string` | `"aws-iam-authenticator"` | no | @@ -243,6 +246,10 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | cluster\_security\_group\_id | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. | | cluster\_version | The Kubernetes server version for the EKS cluster. | | config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. | +| fargate\_iam\_role\_arn | IAM role ARN for EKS Fargate pods | +| fargate\_iam\_role\_name | IAM role name for EKS Fargate pods | +| fargate\_profile\_arns | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | +| fargate\_profile\_ids | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | | kubeconfig | kubectl config file contents for this EKS cluster. | | kubeconfig\_filename | The filename of the generated kubectl config. | | node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | diff --git a/aws_auth.tf b/aws_auth.tf index aa4cf48..91b0a23 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -43,18 +43,20 @@ locals { local.auth_launch_template_worker_roles, local.auth_worker_roles, module.node_groups.aws_auth_roles, + module.fargate.aws_auth_roles, ) : { # Work around https://github.com/kubernetes-sigs/aws-iam-authenticator/issues/153 # Strip the leading slash off so that Terraform doesn't think it's a regex rolearn = replace(role["worker_role_arn"], replace(var.iam_path, "/^//", ""), "") - username = "system:node:{{EC2PrivateDNSName}}" + username = role["platform"] == "fargate" ? "system:node:{{SessionName}}" : "system:node:{{EC2PrivateDNSName}}" groups = tolist(concat( [ "system:bootstrappers", "system:nodes", ], - role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [] + role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [], + role["platform"] == "fargate" ? ["system:node-proxier"] : [], )) } ] diff --git a/examples/fargate/main.tf b/examples/fargate/main.tf new file mode 100644 index 0000000..f3f1aa0 --- /dev/null +++ b/examples/fargate/main.tf @@ -0,0 +1,112 @@ +terraform { + required_version = ">= 0.12.6" +} + +provider "aws" { + version = ">= 2.28.1" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.11" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.47.0" + + name = "test-vpc" + cidr = "172.16.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] + public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + vpc_id = module.vpc.vpc_id + + fargate_profiles = { + example = { + namespace = "default" + + # Kubernetes labels for selection + # labels = { + # Environment = "test" + # GithubRepo = "terraform-aws-eks" + # GithubOrg = "terraform-aws-modules" + # } + + tags = { + Owner = "test" + } + } + } + + map_roles = var.map_roles + map_users = var.map_users + map_accounts = var.map_accounts +} diff --git a/examples/fargate/outputs.tf b/examples/fargate/outputs.tf new file mode 100644 index 0000000..59aa57a --- /dev/null +++ b/examples/fargate/outputs.tf @@ -0,0 +1,29 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "region" { + description = "AWS region." + value = var.region +} + +output "fargate_profile_arns" { + description = "Outputs from node groups" + value = module.eks.fargate_profile_arns +} diff --git a/examples/fargate/variables.tf b/examples/fargate/variables.tf new file mode 100644 index 0000000..7085aea --- /dev/null +++ b/examples/fargate/variables.tf @@ -0,0 +1,52 @@ +variable "region" { + default = "us-west-2" +} + +variable "map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + + default = [ + "777777777777", + "888888888888", + ] +} + +variable "map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + + default = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] +} + +variable "map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + + default = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] +} diff --git a/fargate.tf b/fargate.tf new file mode 100644 index 0000000..6ba03cb --- /dev/null +++ b/fargate.tf @@ -0,0 +1,22 @@ +module "fargate" { + source = "./modules/fargate" + cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] + create_eks = var.create_eks + create_fargate_pod_execution_role = var.create_fargate_pod_execution_role + fargate_pod_execution_role_name = var.fargate_pod_execution_role_name + fargate_profiles = var.fargate_profiles + iam_path = var.iam_path + iam_policy_arn_prefix = local.policy_arn_prefix + subnets = var.subnets + tags = var.tags + + # Hack to ensure ordering of resource creation. + # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 + # Do not create node_groups before other resources are ready and removes race conditions + # Ensure these resources are created before "unlocking" the data source. + # Will be removed in Terraform 0.13 + eks_depends_on = [ + aws_eks_cluster.this, + kubernetes_config_map.aws_auth, + ] +} diff --git a/modules/fargate/README.md b/modules/fargate/README.md new file mode 100644 index 0000000..f22dbb0 --- /dev/null +++ b/modules/fargate/README.md @@ -0,0 +1,54 @@ +# eks `fargate` submodule + +Helper submodule to create and manage resources related to `aws_eks_fargate_profile`. + +## Assumptions +* Designed for use by the parent module and not directly by end users + +## `fargate_profile` keys +`fargate_profile` is a map of maps. Key of first level will be used as unique value for `for_each` resources and in the `aws_eks_fargate_profile` name. Inner map can take the below values. + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| name | Fargate profile name | `string` | Auto generated in the following format `[cluster_name]-fargate-[fargate_profile_map_key]`| no | +| namespace | Kubernetes namespace for selection | `string` | n/a | yes | +| labels | Key-value map of Kubernetes labels for selection | `map(string)` | `{}` | no | +| tags | Key-value map of resource tags. Will be merged with root module tags. | `map(string)` | `var.tags` | no | + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| aws | n/a | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name | Name of the EKS cluster. | `string` | n/a | yes | +| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| create\_fargate\_pod\_execution\_role | Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created. | `bool` | `true` | no | +| eks\_depends\_on | List of references to other resources this submodule depends on. | `any` | `null` | no | +| fargate\_pod\_execution\_role\_name | The IAM Role that provides permissions for the EKS Fargate Profile. | `string` | `null` | no | +| fargate\_profiles | Fargate profiles to create. See `fargate_profile` keys section in README.md for more details | `any` | `{}` | no | +| iam\_path | IAM roles will be created on this path. | `string` | `"/"` | no | +| iam\_policy\_arn\_prefix | IAM policy prefix with the correct AWS partition. | `string` | n/a | yes | +| subnets | A list of subnets for the EKS Fargate profiles. | `list(string)` | `[]` | no | +| tags | A map of tags to add to all resources. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | +| fargate\_profile\_arns | Amazon Resource Name (ARN) of the EKS Fargate Profiles. | +| fargate\_profile\_ids | EKS Cluster name and EKS Fargate Profile names separated by a colon (:). | +| iam\_role\_arn | IAM role ARN for EKS Fargate pods | +| iam\_role\_name | IAM role name for EKS Fargate pods | + + diff --git a/modules/fargate/data.tf b/modules/fargate/data.tf new file mode 100644 index 0000000..fc41f49 --- /dev/null +++ b/modules/fargate/data.tf @@ -0,0 +1,17 @@ +data "aws_iam_policy_document" "eks_fargate_pod_assume_role" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["eks-fargate-pods.amazonaws.com"] + } + } +} + +data "aws_iam_role" "custom_fargate_iam_role" { + count = local.create_eks && ! var.create_fargate_pod_execution_role ? 1 : 0 + name = var.fargate_pod_execution_role_name +} diff --git a/modules/fargate/fargate.tf b/modules/fargate/fargate.tf new file mode 100644 index 0000000..f3592ba --- /dev/null +++ b/modules/fargate/fargate.tf @@ -0,0 +1,29 @@ +resource "aws_iam_role" "eks_fargate_pod" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + name_prefix = format("%s-fargate", var.cluster_name) + assume_role_policy = data.aws_iam_policy_document.eks_fargate_pod_assume_role[0].json + tags = var.tags + path = var.iam_path +} + +resource "aws_iam_role_policy_attachment" "eks_fargate_pod" { + count = local.create_eks && var.create_fargate_pod_execution_role ? 1 : 0 + policy_arn = "${var.iam_policy_arn_prefix}/AmazonEKSFargatePodExecutionRolePolicy" + role = aws_iam_role.eks_fargate_pod[0].name +} + +resource "aws_eks_fargate_profile" "this" { + for_each = local.create_eks ? local.fargate_profiles_expanded : {} + cluster_name = var.cluster_name + fargate_profile_name = lookup(each.value, "name", format("%s-fargate-%s", var.cluster_name, replace(each.key, "_", "-"))) + pod_execution_role_arn = local.pod_execution_role_arn + subnet_ids = var.subnets + tags = each.value.tags + + selector { + namespace = each.value.namespace + labels = lookup(each.value, "labels", null) + } + + depends_on = [var.eks_depends_on] +} diff --git a/modules/fargate/locals.tf b/modules/fargate/locals.tf new file mode 100644 index 0000000..dbd72f0 --- /dev/null +++ b/modules/fargate/locals.tf @@ -0,0 +1,10 @@ +locals { + create_eks = var.create_eks && length(var.fargate_profiles) > 0 + pod_execution_role_arn = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.arn, list("")), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.arn, list("")), 0) + pod_execution_role_name = var.create_fargate_pod_execution_role ? element(concat(aws_iam_role.eks_fargate_pod.*.name, list("")), 0) : element(concat(data.aws_iam_role.custom_fargate_iam_role.*.name, list("")), 0) + + fargate_profiles_expanded = { for k, v in var.fargate_profiles : k => merge( + { tags = var.tags }, + v, + ) if var.create_eks } +} diff --git a/modules/fargate/outputs.tf b/modules/fargate/outputs.tf new file mode 100644 index 0000000..b4a1631 --- /dev/null +++ b/modules/fargate/outputs.tf @@ -0,0 +1,27 @@ +output "fargate_profile_ids" { + description = "EKS Cluster name and EKS Fargate Profile names separated by a colon (:)." + value = [for f in aws_eks_fargate_profile.this : f.id] +} + +output "fargate_profile_arns" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profiles." + value = [for f in aws_eks_fargate_profile.this : f.arn] +} + +output "iam_role_name" { + description = "IAM role name for EKS Fargate pods" + value = local.pod_execution_role_name +} + +output "iam_role_arn" { + description = "IAM role ARN for EKS Fargate pods" + value = local.pod_execution_role_arn +} + +output "aws_auth_roles" { + description = "Roles for use in aws-auth ConfigMap" + value = [{ + worker_role_arn = local.pod_execution_role_arn + platform = "fargate" + }] +} diff --git a/modules/fargate/variables.tf b/modules/fargate/variables.tf new file mode 100644 index 0000000..745f13c --- /dev/null +++ b/modules/fargate/variables.tf @@ -0,0 +1,59 @@ +variable "cluster_name" { + description = "Name of the EKS cluster." + type = string +} + +variable "create_eks" { + description = "Controls if EKS resources should be created (it affects almost all resources)" + type = bool + default = true +} + +variable "iam_path" { + description = "IAM roles will be created on this path." + type = string + default = "/" +} + +variable "iam_policy_arn_prefix" { + description = "IAM policy prefix with the correct AWS partition." + type = string +} + +variable "create_fargate_pod_execution_role" { + description = "Controls if the the IAM Role that provides permissions for the EKS Fargate Profile should be created." + type = bool + default = true +} + +variable "fargate_pod_execution_role_name" { + description = "The IAM Role that provides permissions for the EKS Fargate Profile." + type = string + default = null +} + +variable "fargate_profiles" { + description = "Fargate profiles to create. See `fargate_profile` keys section in README.md for more details" + type = any + default = {} +} + +variable "subnets" { + description = "A list of subnets for the EKS Fargate profiles." + type = list(string) + default = [] +} + +variable "tags" { + description = "A map of tags to add to all resources." + type = map(string) + default = {} +} + +# Hack for a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 +# Will be removed in Terraform 0.13 with the support of module's `depends_on` https://github.com/hashicorp/terraform/issues/10462 +variable "eks_depends_on" { + description = "List of references to other resources this submodule depends on." + type = any + default = null +} diff --git a/outputs.tf b/outputs.tf index 8c71fbe..3cf1452 100644 --- a/outputs.tf +++ b/outputs.tf @@ -166,6 +166,26 @@ output "worker_iam_role_arn" { )[0] } +output "fargate_profile_ids" { + description = "EKS Cluster name and EKS Fargate Profile names separated by a colon (:)." + value = module.fargate.fargate_profile_ids +} + +output "fargate_profile_arns" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profiles." + value = module.fargate.fargate_profile_arns +} + +output "fargate_iam_role_name" { + description = "IAM role name for EKS Fargate pods" + value = module.fargate.iam_role_name +} + +output "fargate_iam_role_arn" { + description = "IAM role ARN for EKS Fargate pods" + value = module.fargate.iam_role_arn +} + output "node_groups" { description = "Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys" value = module.node_groups.node_groups diff --git a/variables.tf b/variables.tf index 965e127..e0675ae 100644 --- a/variables.tf +++ b/variables.tf @@ -349,3 +349,21 @@ variable "cluster_encryption_config" { })) default = [] } + +variable "fargate_profiles" { + description = "Fargate profiles to create. See `fargate_profile` keys section in fargate submodule's README.md for more details" + type = any + default = {} +} + +variable "create_fargate_pod_execution_role" { + description = "Controls if the EKS Fargate pod execution IAM role should be created." + type = bool + default = true +} + +variable "fargate_pod_execution_role_name" { + description = "The IAM Role that provides permissions for the EKS Fargate Profile." + type = string + default = null +}