provider "aws" { region = local.region } provider "helm" { kubernetes { host = module.eks.cluster_endpoint cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) exec { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } } data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" cluster_version = "1.24" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) tags = { Example = local.name GithubRepo = "terraform-aws-eks" GithubOrg = "terraform-aws-modules" } } ################################################################################ # EKS Module ################################################################################ module "eks" { source = "../.." cluster_name = local.name cluster_version = local.cluster_version cluster_endpoint_public_access = true cluster_addons = { kube-proxy = {} vpc-cni = {} } vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets # Fargate profiles use the cluster primary security group so these are not utilized create_cluster_security_group = false create_node_security_group = false fargate_profile_defaults = { iam_role_additional_policies = { additional = aws_iam_policy.additional.arn } } fargate_profiles = { example = { name = "example" selectors = [ { namespace = "backend" labels = { Application = "backend" } }, { namespace = "app-*" labels = { Application = "app-wildcard" } } ] # Using specific subnets instead of the subnets supplied for the cluster itself subnet_ids = [module.vpc.private_subnets[1]] tags = { Owner = "secondary" } timeouts = { create = "20m" delete = "20m" } } kube_system = { name = "kube-system" selectors = [ { namespace = "kube-system" } ] } } tags = local.tags } ################################################################################ # Modify EKS CoreDNS Deployment ################################################################################ data "aws_eks_cluster_auth" "this" { name = module.eks.cluster_name } locals { kubeconfig = yamlencode({ apiVersion = "v1" kind = "Config" current-context = "terraform" clusters = [{ name = module.eks.cluster_name cluster = { certificate-authority-data = module.eks.cluster_certificate_authority_data server = module.eks.cluster_endpoint } }] contexts = [{ name = "terraform" context = { cluster = module.eks.cluster_name user = "terraform" } }] users = [{ name = "terraform" user = { token = data.aws_eks_cluster_auth.this.token } }] }) } # Separate resource so that this is only ever executed once resource "null_resource" "remove_default_coredns_deployment" { triggers = {} provisioner "local-exec" { interpreter = ["/bin/bash", "-c"] environment = { KUBECONFIG = base64encode(local.kubeconfig) } # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon # However, we are maintaing the existing kube-dns service and annotating it for Helm to assume control command = <<-EOT kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) EOT } } resource "null_resource" "modify_kube_dns" { triggers = {} provisioner "local-exec" { interpreter = ["/bin/bash", "-c"] environment = { KUBECONFIG = base64encode(local.kubeconfig) } # We are maintaing the existing kube-dns service and annotating it for Helm to assume control command = <<-EOT echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-namespace=kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) kubectl --namespace kube-system label --overwrite service kube-dns app.kubernetes.io/managed-by=Helm --kubeconfig <(echo $KUBECONFIG | base64 --decode) EOT } depends_on = [ null_resource.remove_default_coredns_deployment ] } ################################################################################ # CoreDNS Helm Chart (self-managed) ################################################################################ data "aws_eks_addon_version" "this" { for_each = toset(["coredns"]) addon_name = each.value kubernetes_version = module.eks.cluster_version most_recent = true } resource "helm_release" "coredns" { name = "coredns" namespace = "kube-system" create_namespace = false description = "CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS Services" chart = "coredns" version = "1.19.4" repository = "https://coredns.github.io/helm" # For EKS image repositories https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html values = [ <<-EOT image: repository: 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns tag: ${data.aws_eks_addon_version.this["coredns"].version} deployment: name: coredns annotations: eks.amazonaws.com/compute-type: fargate service: name: kube-dns annotations: eks.amazonaws.com/compute-type: fargate podAnnotations: eks.amazonaws.com/compute-type: fargate EOT ] depends_on = [ # Need to ensure the CoreDNS updates are peformed before provisioning null_resource.modify_kube_dns ] } ################################################################################ # Supporting Resources ################################################################################ module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "~> 3.0" name = local.name cidr = local.vpc_cidr azs = local.azs private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] enable_nat_gateway = true single_nat_gateway = true enable_dns_hostnames = true enable_flow_log = true create_flow_log_cloudwatch_iam_role = true create_flow_log_cloudwatch_log_group = true public_subnet_tags = { "kubernetes.io/role/elb" = 1 } private_subnet_tags = { "kubernetes.io/role/internal-elb" = 1 } tags = local.tags } resource "aws_iam_policy" "additional" { name = "${local.name}-additional" policy = jsonencode({ Version = "2012-10-17" Statement = [ { Action = [ "ec2:Describe*", ] Effect = "Allow" Resource = "*" }, ] }) }