Support creating multiple worker auto scaling groups, similar to KOPS

README.md

@@ -103,11 +103,13 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a
 | subnets | A list of subnets to associate with the cluster's underlying instances. | list | - | yes |
 | tags | A map of tags to add to all resources. | string | `<map>` | no |
 | vpc_id | VPC id where the cluster and other resources will be deployed. | string | - | yes |
-| workers_ami_id | AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. | string | `` | no |
-| workers_asg_desired_capacity | Desired worker capacity in the autoscaling group. | string | `1` | no |
-| workers_asg_max_size | Maximum worker capacity in the autoscaling group. | string | `3` | no |
-| workers_asg_min_size | Minimum worker capacity in the autoscaling group. | string | `1` | no |
-| workers_instance_type | Size of the workers instances. | string | `m4.large` | no |
+| worker_groups | A list of maps defining worker autoscaling groups | list of maps | - | no |
+| worker_groups.name | Name of the worker group | string | `nodes` | yes
+| worker_groups.ami_id | AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. | string | `` | no |
+| worker_groups.asg_desired_capacity | Desired worker capacity in the autoscaling group. | string | `1` | no |
+| worker_groups.asg_max_size | Maximum worker capacity in the autoscaling group. | string | `3` | no |
+| worker_groups.asg_min_size | Minimum worker capacity in the autoscaling group. | string | `1` | no |
+| worker_groups.instance_type | Size of the workers instances. | string | `m4.large` | no |
 
 ## Outputs
 

data.tf

@@ -1,15 +1,5 @@
 data "aws_region" "current" {}
 
-data "aws_ami" "eks_worker" {
-  filter {
-    name   = "name"
-    values = ["eks-worker-*"]
-  }
-
-  most_recent = true
-  owners      = ["602401143452"] # Amazon
-}
-
 data "aws_iam_policy_document" "workers_assume_role_policy" {
   statement {
     sid = "EKSWorkerAssumeRole"
@@ -40,19 +30,6 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" {
   }
 }
 
-data template_file userdata {
-  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
-
-  vars {
-    region              = "${data.aws_region.current.name}"
-    max_pod_count       = "${lookup(local.max_pod_per_node, var.workers_instance_type)}"
-    cluster_name        = "${var.cluster_name}"
-    endpoint            = "${aws_eks_cluster.this.endpoint}"
-    cluster_auth_base64 = "${aws_eks_cluster.this.certificate_authority.0.data}"
-    additional_userdata = "${var.additional_userdata}"
-  }
-}
-
 data template_file kubeconfig {
   template = "${file("${path.module}/templates/kubeconfig.tpl")}"
 
@@ -71,8 +48,3 @@ data template_file config_map_aws_auth {
     role_arn = "${aws_iam_role.workers.arn}"
   }
 }
-
-module "ebs_optimized" {
-  source        = "./modules/tf_util_ebs_optimized"
-  instance_type = "${var.workers_instance_type}"
-}

local.tf

@@ -1,66 +1,4 @@
 locals {
-  # Mapping from the node type that we selected and the max number of pods that it can run
-  # Taken from https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-06-05/amazon-eks-nodegroup.yaml
-  max_pod_per_node = {
-    c4.large    = 29
-    c4.xlarge   = 58
-    c4.2xlarge  = 58
-    c4.4xlarge  = 234
-    c4.8xlarge  = 234
-    c5.large    = 29
-    c5.xlarge   = 58
-    c5.2xlarge  = 58
-    c5.4xlarge  = 234
-    c5.9xlarge  = 234
-    c5.18xlarge = 737
-    i3.large    = 29
-    i3.xlarge   = 58
-    i3.2xlarge  = 58
-    i3.4xlarge  = 234
-    i3.8xlarge  = 234
-    i3.16xlarge = 737
-    m3.medium   = 12
-    m3.large    = 29
-    m3.xlarge   = 58
-    m3.2xlarge  = 118
-    m4.large    = 20
-    m4.xlarge   = 58
-    m4.2xlarge  = 58
-    m4.4xlarge  = 234
-    m4.10xlarge = 234
-    m5.large    = 29
-    m5.xlarge   = 58
-    m5.2xlarge  = 58
-    m5.4xlarge  = 234
-    m5.12xlarge = 234
-    m5.24xlarge = 737
-    p2.xlarge   = 58
-    p2.8xlarge  = 234
-    p2.16xlarge = 234
-    p3.2xlarge  = 58
-    p3.8xlarge  = 234
-    p3.16xlarge = 234
-    r3.xlarge   = 58
-    r3.2xlarge  = 58
-    r3.4xlarge  = 234
-    r3.8xlarge  = 234
-    r4.large    = 29
-    r4.xlarge   = 58
-    r4.2xlarge  = 58
-    r4.4xlarge  = 234
-    r4.8xlarge  = 234
-    r4.16xlarge = 737
-    t2.small    = 8
-    t2.medium   = 17
-    t2.large    = 35
-    t2.xlarge   = 44
-    t2.2xlarge  = 44
-    x1.16xlarge = 234
-    x1.32xlarge = 234
-  }
-
-  asg_tags = ["${null_resource.tags_as_list_of_maps.*.triggers}"]
-
   # More information: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-06-05/amazon-eks-nodegroup.yaml
   config_map_aws_auth = <<CONFIGMAPAWSAUTH
 apiVersion: v1

modules/tf_util_ebs_optimized/outputs.tf

@@ -1,4 +0,0 @@
-output "answer" {
-  description = "Returns true or false depending on if the instance type is able to be EBS optimized."
-  value       = "${lookup(local.ebs_optimized_types, var.instance_type, false)}"
-}

modules/tf_util_ebs_optimized/variables.tf

@@ -1,3 +0,0 @@
-variable "instance_type" {
-  description = "Instance type to evaluate if EBS optimized is an option."
-}

modules/worker_groups/data.tf

@@ -0,0 +1,9 @@
+data "aws_ami" "eks_worker" {
+  filter {
+    name   = "name"
+    values = ["eks-worker-*"]
+  }
+
+  most_recent = true
+  owners      = ["602401143452"] # Amazon
+}

modules/worker_groups/local.tf

@@ -1,10 +1,66 @@
-/**
-# terraform_util_ebs_optimized
-
-A terraform module to return true or false based on if an instance type supports the EBS optmized flag.
-*/
-
 locals {
+  asg_tags = ["${null_resource.tags_as_list_of_maps.*.triggers}"]
+
+  # Mapping from the node type that we selected and the max number of pods that it can run
+  # Taken from https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-06-05/amazon-eks-nodegroup.yaml
+  max_pod_per_node = {
+    c4.large    = 29
+    c4.xlarge   = 58
+    c4.2xlarge  = 58
+    c4.4xlarge  = 234
+    c4.8xlarge  = 234
+    c5.large    = 29
+    c5.xlarge   = 58
+    c5.2xlarge  = 58
+    c5.4xlarge  = 234
+    c5.9xlarge  = 234
+    c5.18xlarge = 737
+    i3.large    = 29
+    i3.xlarge   = 58
+    i3.2xlarge  = 58
+    i3.4xlarge  = 234
+    i3.8xlarge  = 234
+    i3.16xlarge = 737
+    m3.medium   = 12
+    m3.large    = 29
+    m3.xlarge   = 58
+    m3.2xlarge  = 118
+    m4.large    = 20
+    m4.xlarge   = 58
+    m4.2xlarge  = 58
+    m4.4xlarge  = 234
+    m4.10xlarge = 234
+    m5.large    = 29
+    m5.xlarge   = 58
+    m5.2xlarge  = 58
+    m5.4xlarge  = 234
+    m5.12xlarge = 234
+    m5.24xlarge = 737
+    p2.xlarge   = 58
+    p2.8xlarge  = 234
+    p2.16xlarge = 234
+    p3.2xlarge  = 58
+    p3.8xlarge  = 234
+    p3.16xlarge = 234
+    r3.xlarge   = 58
+    r3.2xlarge  = 58
+    r3.4xlarge  = 234
+    r3.8xlarge  = 234
+    r4.large    = 29
+    r4.xlarge   = 58
+    r4.2xlarge  = 58
+    r4.4xlarge  = 234
+    r4.8xlarge  = 234
+    r4.16xlarge = 737
+    t2.small    = 8
+    t2.medium   = 17
+    t2.large    = 35
+    t2.xlarge   = 44
+    t2.2xlarge  = 44
+    x1.16xlarge = 234
+    x1.32xlarge = 234
+  }
+
   ebs_optimized_types = {
     "c4.large"     = true
     "c4.xlarge"    = true

modules/worker_groups/main.tf

@@ -0,0 +1,64 @@
+resource "aws_autoscaling_group" "workers" {
+  count = "${length(var.worker_groups)}"
+
+  name_prefix          = "${lookup(var.worker_groups[count.index], "name")}.${var.cluster_name}"
+  launch_configuration = "${element(aws_launch_configuration.workers.*.id, count.index)}"
+  desired_capacity     = "${lookup(var.worker_groups[count.index], "asg_desired_capacity")}"
+  max_size             = "${lookup(var.worker_groups[count.index], "asg_max_size")}"
+  min_size             = "${lookup(var.worker_groups[count.index], "asg_min_size")}"
+  vpc_zone_identifier  = ["${var.subnets}"]
+
+  tags = ["${concat(
+    list(
+      map("key", "Name", "value", "${lookup(var.worker_groups[count.index], "name")}.${var.cluster_name}-eks_asg", "propagate_at_launch", true),
+      map("key", "kubernetes.io/cluster/${var.cluster_name}", "value", "owned", "propagate_at_launch", true),
+    ),
+    local.asg_tags)
+  }"]
+}
+
+resource "aws_launch_configuration" "workers" {
+  count = "${length(var.worker_groups)}"
+
+  name_prefix                 = "${lookup(var.worker_groups[count.index], "name")}.${lookup(var.worker_groups[count.index], "name")}.${var.cluster_name}"
+  associate_public_ip_address = true
+  iam_instance_profile        = "${var.iam_instance_profile}"
+  image_id                    = "${lookup(var.worker_groups[count.index], "ami_id") == "" ? data.aws_ami.eks_worker.id : lookup(var.worker_groups[count.index], "ami_id")}"
+  instance_type               = "${lookup(var.worker_groups[count.index], "instance_type")}"
+  security_groups             = ["${var.security_group_id}"]
+  user_data_base64            = "${base64encode(element(data.template_file.userdata.*.rendered, count.index))}"
+  ebs_optimized               = "${var.ebs_optimized_workers ? lookup(local.ebs_optimized_types, lookup(var.worker_groups[count.index], "instance_type"), false) : false}"
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  root_block_device {
+    delete_on_termination = true
+  }
+}
+
+data template_file userdata {
+  count = "${length(var.worker_groups)}"
+
+  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
+
+  vars {
+    region              = "${var.aws_region}"
+    max_pod_count       = "${lookup(local.max_pod_per_node, lookup(var.worker_groups[count.index], "instance_type"))}"
+    cluster_name        = "${var.cluster_name}"
+    endpoint            = "${var.endpoint}"
+    cluster_auth_base64 = "${var.certificate_authority}"
+    additional_userdata = "${var.additional_userdata}"
+  }
+}
+
+resource "null_resource" "tags_as_list_of_maps" {
+  count = "${length(keys(var.tags))}"
+
+  triggers = "${map(
+    "key", "${element(keys(var.tags), count.index)}",
+    "value", "${element(values(var.tags), count.index)}",
+    "propagate_at_launch", "true"
+  )}"
+}

modules/worker_groups/variables.tf

@@ -0,0 +1,64 @@
+variable "additional_userdata" {
+  description = "Extra lines of userdata (bash) which are appended to the default userdata code."
+  default     = ""
+}
+
+variable "aws_region" {
+  description = "The AWS region where the cluster resides."
+}
+
+variable "certificate_authority" {
+  description = "Base64 encoded certificate authority of the cluster."
+}
+
+variable "cluster_name" {
+  description = "Name of the EKS cluster which is also used as a prefix in names of related resources."
+}
+
+variable "ebs_optimized_workers" {
+  description = "If left at default of true, will use ebs optimization if available on the given instance type."
+  default     = true
+}
+
+variable "endpoint" {
+  description = "API endpoint of the cluster."
+}
+
+variable "iam_instance_profile" {
+  description = "Worker IAM instance profile name."
+}
+
+variable "security_group_id" {
+  description = "Worker security group ID."
+}
+
+variable "subnets" {
+  description = "A list of subnets to associate with the cluster's underlying instances."
+  type        = "list"
+}
+
+variable "tags" {
+  description = "A map of tags to add to all resources."
+  default     = {}
+}
+
+variable "workers_ami_id" {
+  description = "AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI."
+  default     = ""
+}
+
+variable "worker_groups" {
+  description = "A list of maps defining worker group configurations."
+  type        = "list"
+
+  default = [
+    {
+      name                 = "nodes"    # Name of the worker group.
+      ami_id               = ""         # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI.
+      asg_desired_capacity = "1"        # Desired worker capacity in the autoscaling group.
+      asg_max_size         = "3"        # Maximum worker capacity in the autoscaling group.
+      asg_min_size         = "1"        # Minimum worker capacity in the autoscaling group.
+      instance_type        = "m4.large" # Size of the workers instances.
+    },
+  ]
+}

outputs.tf

@@ -20,7 +20,7 @@ output "cluster_id" {
 # }
 
 output "cluster_certificate_authority_data" {
-  description = "Nested attribute containing certificate-authority-data for your cluster. Tis is the base64 encoded certificate data required to communicate with your cluster."
+  description = "Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster."
   value       = "${aws_eks_cluster.this.certificate_authority.0.data}"
 }
 

variables.tf

@@ -46,27 +46,18 @@ variable "vpc_id" {
   description = "VPC id where the cluster and other resources will be deployed."
 }
 
-variable "workers_ami_id" {
-  description = "AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI."
-  default     = ""
-}
+variable "worker_groups" {
+  description = "A list of maps defining worker group configurations."
+  type        = "list"
 
-variable "workers_asg_desired_capacity" {
-  description = "Desired worker capacity in the autoscaling group."
-  default     = "1"
-}
-
-variable "workers_asg_max_size" {
-  description = "Maximum worker capacity in the autoscaling group."
-  default     = "3"
-}
-
-variable "workers_asg_min_size" {
-  description = "Minimum worker capacity in the autoscaling group."
-  default     = "1"
-}
-
-variable "workers_instance_type" {
-  description = "Size of the workers instances."
-  default     = "m4.large"
+  default = [
+    {
+      name                 = "nodes"    # Name of the worker group.
+      ami_id               = ""         # AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI.
+      asg_desired_capacity = "1"        # Desired worker capacity in the autoscaling group.
+      asg_max_size         = "3"        # Maximum worker capacity in the autoscaling group.
+      asg_min_size         = "1"        # Minimum worker capacity in the autoscaling group.
+      instance_type        = "m4.large" # Size of the workers instances.
+    },
+  ]
 }

workers.tf

@@ -1,37 +1,14 @@
-resource "aws_autoscaling_group" "workers" {
-  name_prefix          = "${var.cluster_name}"
-  launch_configuration = "${aws_launch_configuration.workers.id}"
-  desired_capacity     = "${var.workers_asg_desired_capacity}"
-  max_size             = "${var.workers_asg_max_size}"
-  min_size             = "${var.workers_asg_min_size}"
-  vpc_zone_identifier  = ["${var.subnets}"]
-
-  tags = ["${concat(
-    list(
-      map("key", "Name", "value", "${var.cluster_name}-eks_asg", "propagate_at_launch", true),
-      map("key", "kubernetes.io/cluster/${var.cluster_name}", "value", "owned", "propagate_at_launch", true),
-    ),
-    local.asg_tags)
-  }"]
-}
-
-resource "aws_launch_configuration" "workers" {
-  name_prefix                 = "${var.cluster_name}"
-  associate_public_ip_address = true
-  iam_instance_profile        = "${aws_iam_instance_profile.workers.name}"
-  image_id                    = "${var.workers_ami_id == "" ? data.aws_ami.eks_worker.id : var.workers_ami_id}"
-  instance_type               = "${var.workers_instance_type}"
-  security_groups             = ["${aws_security_group.workers.id}"]
-  user_data_base64            = "${base64encode(data.template_file.userdata.rendered)}"
-  ebs_optimized               = "${var.ebs_optimized_workers ? module.ebs_optimized.answer : false}"
-
-  lifecycle {
-    create_before_destroy = true
-  }
-
-  root_block_device {
-    delete_on_termination = true
-  }
+module "worker_groups" {
+  source                = "./modules/worker_groups"
+  aws_region            = "{data.aws_region.current.name}"
+  cluster_name          = "${var.cluster_name}"
+  certificate_authority = "${aws_eks_cluster.this.certificate_authority.0.data}"
+  endpoint              = "${aws_eks_cluster.this.endpoint}"
+  iam_instance_profile  = "${aws_iam_instance_profile.workers.name}"
+  security_group_id     = "${aws_security_group.workers.id}"
+  subnets               = "${var.subnets}"
+  tags                  = "${var.tags}"
+  worker_groups         = "${var.worker_groups}"
 }
 
 resource "aws_security_group" "workers" {
@@ -96,13 +73,3 @@ resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryRea
   policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
   role       = "${aws_iam_role.workers.name}"
 }
-
-resource "null_resource" "tags_as_list_of_maps" {
-  count = "${length(keys(var.tags))}"
-
-  triggers = "${map(
-    "key", "${element(keys(var.tags), count.index)}",
-    "value", "${element(values(var.tags), count.index)}",
-    "propagate_at_launch", "true"
-  )}"
-}