diff --git a/README.md b/README.md index 4136b60..44fbcfe 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | terraform | >= 0.12.9, != 0.13.0 | -| aws | >= 2.55.0 | +| aws | >= 3.3.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | @@ -156,7 +156,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| aws | >= 2.55.0 | +| aws | >= 3.3.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | diff --git a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf new file mode 100644 index 0000000..bfeb9e8 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf @@ -0,0 +1,77 @@ +// if you have used ASGs before, that role got auto-created already and you need to import to TF state +resource "aws_iam_service_linked_role" "autoscaling" { + aws_service_name = "autoscaling.amazonaws.com" + description = "Default Service-Linked Role enables access to AWS Services and Resources used or managed by Auto Scaling" +} + +data "aws_caller_identity" "current" {} + +// This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes +data "aws_iam_policy_document" "ebs_decryption" { + // copy of default KMS policy that lets you manage it + statement { + sid = "Enable IAM User Permissions" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + + actions = [ + "kms:*" + ] + + resources = ["*"] + } + + // required for EKS + statement { + sid = "Allow service-linked role use of the CMK" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ] + + resources = ["*"] + } + + statement { + sid = "Allow attachment of persistent resources" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + actions = [ + "kms:CreateGrant" + ] + + resources = ["*"] + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = ["true"] + } + + } +} diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf new file mode 100644 index 0000000..390e91d --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -0,0 +1,89 @@ +data "template_file" "launch_template_userdata" { + template = file("${path.module}/templates/userdata.sh.tpl") + + vars = { + cluster_name = local.cluster_name + endpoint = module.eks.cluster_endpoint + cluster_auth_base64 = module.eks.cluster_certificate_authority_data + + bootstrap_extra_args = "" + kubelet_extra_args = "" + } +} + +// this is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) +// there are several more options one could set but you probably dont need to modify them +// you can take the default and add your custom AMI and/or custom tags +// +// Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, +// then the default user-data for bootstrapping a cluster is merged in the copy. +resource "aws_launch_template" "default" { + name_prefix = "eks-example-" + description = "Default Launch-Template" + update_default_version = true + + block_device_mappings { + device_name = "/dev/xvda" + + ebs { + volume_size = 100 + volume_type = "gp2" + delete_on_termination = true + //encrypted = true + // enable this if you want to encrypt your node root volumes with a KMS/CMK. encryption of PVCs is handled via k8s StorageClass tho + // you also need to attach data.aws_iam_policy_document.ebs_decryption.json from the disk_encryption_policy.tf to the KMS/CMK key then !! + //kms_key_id = var.kms_key_arn + } + } + + instance_type = var.instance_type + + monitoring { + enabled = true + } + + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + security_groups = [module.eks.worker_security_group_id] + } + + //image_id = var.ami_id // if you want to use a custom AMI + + // if you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then + // you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 + // + // (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) + + // user_data = base64encode( + // data.template_file.launch_template_userdata.rendered, + // ) + + + // supplying custom tags to EKS instances is another use-case for LaunchTemplates + tag_specifications { + resource_type = "instance" + + tags = { + CustomTag = "EKS example" + } + } + + // supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) + tag_specifications { + resource_type = "volume" + + tags = { + CustomTag = "EKS example" + } + } + + // tag the LT itself + tags = { + CustomTag = "EKS example" + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf new file mode 100644 index 0000000..a20b382 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -0,0 +1,93 @@ +terraform { + required_version = ">= 0.12.9" +} + +provider "aws" { + version = ">= 3.3.0" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~>1.4" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.11" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-lt-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.47.0" + + name = "test-vpc" + cidr = "172.16.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] + public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" // EKS adds this and TF would want to remove then later + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + + node_groups = { + example = { + desired_capacity = 1 + max_capacity = 15 + min_capacity = 1 + + launch_template_id = aws_launch_template.default.id + launch_template_version = aws_launch_template.default.default_version + + additional_tags = { + CustomTag = "EKS example" + } + } + } +} diff --git a/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl new file mode 100644 index 0000000..6cbad79 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl @@ -0,0 +1,12 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" +#!/bin/bash +set -xe + +# Bootstrap and join the cluster +/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" '${cluster_name}' + +--//-- diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf new file mode 100644 index 0000000..2d98686 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + default = "eu-central-1" +} + +variable "instance_type" { + default = "t3.small" // smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS + type = string +} + +variable "kms_key_arn" { + default = "" + description = "KMS key ARN to use if you want to encrypt EKS node root volumes" + type = string +} diff --git a/local.tf b/local.tf index 2554697..cd6ac7e 100644 --- a/local.tf +++ b/local.tf @@ -80,6 +80,7 @@ locals { # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_id = "" # The id of the launch template used for managed node_groups launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group launch_template_placement_tenancy = "default" # The placement tenancy for instances launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index c905c74..9596222 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -26,6 +26,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | instance\_type | Workers' instance type | string | `var.workers_group_defaults[instance_type]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | +| launch_template_id | The id of a aws_launch_template to use | string | No LT used | +| launch\_template_version | The version of the LT to use | string | none | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | | min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | | name | Name of the node group | string | Auto generated | diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 43cf672..222412d 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -2,13 +2,15 @@ locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( { - desired_capacity = var.workers_group_defaults["asg_desired_capacity"] - iam_role_arn = var.default_iam_role_arn - instance_type = var.workers_group_defaults["instance_type"] - key_name = var.workers_group_defaults["key_name"] - max_capacity = var.workers_group_defaults["asg_max_size"] - min_capacity = var.workers_group_defaults["asg_min_size"] - subnets = var.workers_group_defaults["subnets"] + desired_capacity = var.workers_group_defaults["asg_desired_capacity"] + iam_role_arn = var.default_iam_role_arn + instance_type = var.workers_group_defaults["instance_type"] + key_name = var.workers_group_defaults["key_name"] + launch_template_id = var.workers_group_defaults["launch_template_id"] + launch_template_version = var.workers_group_defaults["launch_template_version"] + max_capacity = var.workers_group_defaults["asg_max_size"] + min_capacity = var.workers_group_defaults["asg_min_size"] + subnets = var.workers_group_defaults["subnets"] }, var.node_groups_defaults, v, diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 77fa02e..ba7e265 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -15,7 +15,7 @@ resource "aws_eks_node_group" "workers" { ami_type = lookup(each.value, "ami_type", null) disk_size = lookup(each.value, "disk_size", null) - instance_types = [each.value["instance_type"]] + instance_types = each.value["launch_template_id"] != "" ? [] : [each.value["instance_type"]] release_version = lookup(each.value, "ami_release_version", null) dynamic "remote_access" { @@ -30,6 +30,18 @@ resource "aws_eks_node_group" "workers" { } } + dynamic "launch_template" { + for_each = each.value["launch_template_id"] != "" ? [{ + id = each.value["launch_template_id"] + version = each.value["launch_template_version"] + }] : [] + + content { + id = launch_template.value["id"] + version = launch_template.value["version"] + } + } + version = lookup(each.value, "version", null) labels = merge( diff --git a/modules/node_groups/random.tf b/modules/node_groups/random.tf index 16c0583..aae2c6d 100644 --- a/modules/node_groups/random.tf +++ b/modules/node_groups/random.tf @@ -17,6 +17,7 @@ resource "random_pet" "node_groups" { )) subnet_ids = join("|", each.value["subnets"]) node_group_name = join("-", [var.cluster_name, each.key]) + launch_template = lookup(each.value, "launch_template_id", null) } depends_on = [var.ng_depends_on] diff --git a/versions.tf b/versions.tf index 27c659f..83458a4 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9, != 0.13.0" required_providers { - aws = ">= 2.55.0" + aws = ">= 3.3.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1"