From 127a3a883189d887eac3d0c9e91b7aa335d53b77 Mon Sep 17 00:00:00 2001 From: philicious Date: Mon, 2 Nov 2020 08:19:10 +0100 Subject: [PATCH] feat: Add Launch Template support for Managed Node Groups (#997) NOTES: Managed Node Groups now support Launch Templates. The Launch Template it self is not managed by this module, so you have to create it by your self and pass it's id to this module. See docs and [`examples/launch_templates_with_managed_node_groups/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/launch_templates_with_managed_node_group) for more details. --- README.md | 4 +- .../disk_encryption_policy.tf | 77 +++++++++++++++ .../launchtemplate.tf | 89 ++++++++++++++++++ .../main.tf | 93 +++++++++++++++++++ .../templates/userdata.sh.tpl | 12 +++ .../variables.tf | 14 +++ local.tf | 1 + modules/node_groups/README.md | 2 + modules/node_groups/locals.tf | 16 ++-- modules/node_groups/node_groups.tf | 14 ++- modules/node_groups/random.tf | 1 + versions.tf | 2 +- 12 files changed, 314 insertions(+), 11 deletions(-) create mode 100644 examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf create mode 100644 examples/launch_templates_with_managed_node_groups/launchtemplate.tf create mode 100644 examples/launch_templates_with_managed_node_groups/main.tf create mode 100644 examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl create mode 100644 examples/launch_templates_with_managed_node_groups/variables.tf diff --git a/README.md b/README.md index 4136b60..44fbcfe 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| | terraform | >= 0.12.9, != 0.13.0 | -| aws | >= 2.55.0 | +| aws | >= 3.3.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | @@ -156,7 +156,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Version | |------|---------| -| aws | >= 2.55.0 | +| aws | >= 3.3.0 | | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | diff --git a/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf new file mode 100644 index 0000000..bfeb9e8 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/disk_encryption_policy.tf @@ -0,0 +1,77 @@ +// if you have used ASGs before, that role got auto-created already and you need to import to TF state +resource "aws_iam_service_linked_role" "autoscaling" { + aws_service_name = "autoscaling.amazonaws.com" + description = "Default Service-Linked Role enables access to AWS Services and Resources used or managed by Auto Scaling" +} + +data "aws_caller_identity" "current" {} + +// This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes +data "aws_iam_policy_document" "ebs_decryption" { + // copy of default KMS policy that lets you manage it + statement { + sid = "Enable IAM User Permissions" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + + actions = [ + "kms:*" + ] + + resources = ["*"] + } + + // required for EKS + statement { + sid = "Allow service-linked role use of the CMK" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ] + + resources = ["*"] + } + + statement { + sid = "Allow attachment of persistent resources" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", // required for the ASG to manage encrypted volumes for nodes + module.eks.cluster_iam_role_arn, // required for the cluster / persistentvolume-controller to create encrypted PVCs + ] + } + + actions = [ + "kms:CreateGrant" + ] + + resources = ["*"] + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = ["true"] + } + + } +} diff --git a/examples/launch_templates_with_managed_node_groups/launchtemplate.tf b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf new file mode 100644 index 0000000..390e91d --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/launchtemplate.tf @@ -0,0 +1,89 @@ +data "template_file" "launch_template_userdata" { + template = file("${path.module}/templates/userdata.sh.tpl") + + vars = { + cluster_name = local.cluster_name + endpoint = module.eks.cluster_endpoint + cluster_auth_base64 = module.eks.cluster_certificate_authority_data + + bootstrap_extra_args = "" + kubelet_extra_args = "" + } +} + +// this is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) +// there are several more options one could set but you probably dont need to modify them +// you can take the default and add your custom AMI and/or custom tags +// +// Trivia: AWS transparently creates a copy of your LaunchTemplate and actually uses that copy then for the node group. If you DONT use a custom AMI, +// then the default user-data for bootstrapping a cluster is merged in the copy. +resource "aws_launch_template" "default" { + name_prefix = "eks-example-" + description = "Default Launch-Template" + update_default_version = true + + block_device_mappings { + device_name = "/dev/xvda" + + ebs { + volume_size = 100 + volume_type = "gp2" + delete_on_termination = true + //encrypted = true + // enable this if you want to encrypt your node root volumes with a KMS/CMK. encryption of PVCs is handled via k8s StorageClass tho + // you also need to attach data.aws_iam_policy_document.ebs_decryption.json from the disk_encryption_policy.tf to the KMS/CMK key then !! + //kms_key_id = var.kms_key_arn + } + } + + instance_type = var.instance_type + + monitoring { + enabled = true + } + + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + security_groups = [module.eks.worker_security_group_id] + } + + //image_id = var.ami_id // if you want to use a custom AMI + + // if you use a custom AMI, you need to supply via user-data, the bootstrap script as EKS DOESNT merge its managed user-data then + // you can add more than the minimum code you see in the template, e.g. install SSM agent, see https://github.com/aws/containers-roadmap/issues/593#issuecomment-577181345 + // + // (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) + + // user_data = base64encode( + // data.template_file.launch_template_userdata.rendered, + // ) + + + // supplying custom tags to EKS instances is another use-case for LaunchTemplates + tag_specifications { + resource_type = "instance" + + tags = { + CustomTag = "EKS example" + } + } + + // supplying custom tags to EKS instances root volumes is another use-case for LaunchTemplates. (doesnt add tags to dynamically provisioned volumes via PVC tho) + tag_specifications { + resource_type = "volume" + + tags = { + CustomTag = "EKS example" + } + } + + // tag the LT itself + tags = { + CustomTag = "EKS example" + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/examples/launch_templates_with_managed_node_groups/main.tf b/examples/launch_templates_with_managed_node_groups/main.tf new file mode 100644 index 0000000..a20b382 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/main.tf @@ -0,0 +1,93 @@ +terraform { + required_version = ">= 0.12.9" +} + +provider "aws" { + version = ">= 3.3.0" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~>1.4" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_eks_cluster" "cluster" { + name = module.eks.cluster_id +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false + version = "~> 1.11" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-lt-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.47.0" + + name = "test-vpc" + cidr = "172.16.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] + public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" // EKS adds this and TF would want to remove then later + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + cluster_version = "1.17" + subnets = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + + node_groups = { + example = { + desired_capacity = 1 + max_capacity = 15 + min_capacity = 1 + + launch_template_id = aws_launch_template.default.id + launch_template_version = aws_launch_template.default.default_version + + additional_tags = { + CustomTag = "EKS example" + } + } + } +} diff --git a/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl new file mode 100644 index 0000000..6cbad79 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/templates/userdata.sh.tpl @@ -0,0 +1,12 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" +#!/bin/bash +set -xe + +# Bootstrap and join the cluster +/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" '${cluster_name}' + +--//-- diff --git a/examples/launch_templates_with_managed_node_groups/variables.tf b/examples/launch_templates_with_managed_node_groups/variables.tf new file mode 100644 index 0000000..2d98686 --- /dev/null +++ b/examples/launch_templates_with_managed_node_groups/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + default = "eu-central-1" +} + +variable "instance_type" { + default = "t3.small" // smallest recommended, where ~1.1Gb of 2Gb memory is available for the Kubernetes pods after ‘warming up’ Docker, Kubelet, and OS + type = string +} + +variable "kms_key_arn" { + default = "" + description = "KMS key ARN to use if you want to encrypt EKS node root volumes" + type = string +} diff --git a/local.tf b/local.tf index 2554697..cd6ac7e 100644 --- a/local.tf +++ b/local.tf @@ -80,6 +80,7 @@ locals { # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_id = "" # The id of the launch template used for managed node_groups launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group launch_template_placement_tenancy = "default" # The placement tenancy for instances launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index c905c74..9596222 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -26,6 +26,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | instance\_type | Workers' instance type | string | `var.workers_group_defaults[instance_type]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | +| launch_template_id | The id of a aws_launch_template to use | string | No LT used | +| launch\_template_version | The version of the LT to use | string | none | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | | min\_capacity | Min number of workers | number | `var.workers_group_defaults[asg_min_size]` | | name | Name of the node group | string | Auto generated | diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 43cf672..222412d 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -2,13 +2,15 @@ locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( { - desired_capacity = var.workers_group_defaults["asg_desired_capacity"] - iam_role_arn = var.default_iam_role_arn - instance_type = var.workers_group_defaults["instance_type"] - key_name = var.workers_group_defaults["key_name"] - max_capacity = var.workers_group_defaults["asg_max_size"] - min_capacity = var.workers_group_defaults["asg_min_size"] - subnets = var.workers_group_defaults["subnets"] + desired_capacity = var.workers_group_defaults["asg_desired_capacity"] + iam_role_arn = var.default_iam_role_arn + instance_type = var.workers_group_defaults["instance_type"] + key_name = var.workers_group_defaults["key_name"] + launch_template_id = var.workers_group_defaults["launch_template_id"] + launch_template_version = var.workers_group_defaults["launch_template_version"] + max_capacity = var.workers_group_defaults["asg_max_size"] + min_capacity = var.workers_group_defaults["asg_min_size"] + subnets = var.workers_group_defaults["subnets"] }, var.node_groups_defaults, v, diff --git a/modules/node_groups/node_groups.tf b/modules/node_groups/node_groups.tf index 77fa02e..ba7e265 100644 --- a/modules/node_groups/node_groups.tf +++ b/modules/node_groups/node_groups.tf @@ -15,7 +15,7 @@ resource "aws_eks_node_group" "workers" { ami_type = lookup(each.value, "ami_type", null) disk_size = lookup(each.value, "disk_size", null) - instance_types = [each.value["instance_type"]] + instance_types = each.value["launch_template_id"] != "" ? [] : [each.value["instance_type"]] release_version = lookup(each.value, "ami_release_version", null) dynamic "remote_access" { @@ -30,6 +30,18 @@ resource "aws_eks_node_group" "workers" { } } + dynamic "launch_template" { + for_each = each.value["launch_template_id"] != "" ? [{ + id = each.value["launch_template_id"] + version = each.value["launch_template_version"] + }] : [] + + content { + id = launch_template.value["id"] + version = launch_template.value["version"] + } + } + version = lookup(each.value, "version", null) labels = merge( diff --git a/modules/node_groups/random.tf b/modules/node_groups/random.tf index 16c0583..aae2c6d 100644 --- a/modules/node_groups/random.tf +++ b/modules/node_groups/random.tf @@ -17,6 +17,7 @@ resource "random_pet" "node_groups" { )) subnet_ids = join("|", each.value["subnets"]) node_group_name = join("-", [var.cluster_name, each.key]) + launch_template = lookup(each.value, "launch_template_id", null) } depends_on = [var.ng_depends_on] diff --git a/versions.tf b/versions.tf index 27c659f..83458a4 100644 --- a/versions.tf +++ b/versions.tf @@ -2,7 +2,7 @@ terraform { required_version = ">= 0.12.9, != 0.13.0" required_providers { - aws = ">= 2.55.0" + aws = ">= 3.3.0" local = ">= 1.4" null = ">= 2.1" template = ">= 2.1"