From 28f7e9dd41579820684896ee624ff59963875f56 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Mon, 13 Aug 2018 10:04:02 +0200 Subject: [PATCH 1/6] initial commit --- README.md | 5 +++- data.tf | 2 +- docs/autoscaling.md | 25 +++++++++++++++++++ main.tf | 4 ++++ variables.tf | 7 +----- workers.tf | 58 +++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 docs/autoscaling.md diff --git a/README.md b/README.md index 9f45a61..781cd6d 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ module "eks" { } ``` +## Other documentation + +- [Autoscaling](docs/autoscaling.md): How to enabled worker node autoscaling. + ## Release schedule Generally the maintainers will try to release the module once every 2 weeks to @@ -109,7 +113,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | subnets | A list of subnets to place the EKS cluster and workers within. | list | - | yes | | tags | A map of tags to add to all resources. | map | `` | no | | vpc_id | VPC where the cluster and workers will be deployed. | string | - | yes | -| worker_group_count | The number of maps contained within the worker_groups list. | string | `1` | no | | worker_groups | A list of maps defining worker group configurations. See workers_group_defaults for valid keys. | list | `` | no | | worker_security_group_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `` | no | | worker_sg_ingress_from_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | string | `1025` | no | diff --git a/data.tf b/data.tf index 3671d1a..a04e894 100644 --- a/data.tf +++ b/data.tf @@ -71,7 +71,7 @@ EOF data "template_file" "userdata" { template = "${file("${path.module}/templates/userdata.sh.tpl")}" - count = "${var.worker_group_count}" + count = "${length(var.worker_groups)}" vars { region = "${data.aws_region.current.name}" diff --git a/docs/autoscaling.md b/docs/autoscaling.md new file mode 100644 index 0000000..14e1fd7 --- /dev/null +++ b/docs/autoscaling.md @@ -0,0 +1,25 @@ +# Autoscaling + +Autoscaling of worker nodes can be easily enabled by setting the `autoscaling_enabled` variable to `true` for a worker group in the `worker_groups` map. +This will add the required tags to the autoscaling group for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler). + +You will also need to install the cluster-autoscaler into your cluster. The easiest way to do this is with [helm](https://helm.sh/). + +The [helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler) for the cluster-autoscaler requires some specific settings to work in an EKS cluster. These settings are supplied via YAML values file when installing the helm chart. Here is an example values file: + +```yaml +rbac: + create: true + +sslCertPath: /etc/ssl/certs/ca-bundle.crt + +autoDiscovery: + clusterName: YOUR_CLUSTER_NAME + enabled: true +``` + +To install the chart, simply run helm with the `--values` option: + +``` +helm install stable/cluster-autoscaler --values=path/to/your/values-file.yaml +``` diff --git a/main.tf b/main.tf index efa289f..872eec3 100644 --- a/main.tf +++ b/main.tf @@ -32,6 +32,10 @@ * } * ``` +* ## Other documentation +* +* - [Autoscaling](docs/autoscaling.md): How to enabled worker node autoscaling. + * ## Release schedule * Generally the maintainers will try to release the module once every 2 weeks to diff --git a/variables.tf b/variables.tf index b2363da..bc672b6 100644 --- a/variables.tf +++ b/variables.tf @@ -69,12 +69,6 @@ variable "worker_groups" { }] } -variable "worker_group_count" { - description = "The number of maps contained within the worker_groups list." - type = "string" - default = "1" -} - variable "workers_group_defaults" { description = "Default values for target groups as defined by the list of maps." type = "map" @@ -98,6 +92,7 @@ variable "workers_group_defaults" { public_ip = false # Associate a public ip address with a worker kubelet_node_labels = "" # This string is passed directly to kubelet via --node-labels= if set. It should be comma delimited with no spaces. If left empty no --node-labels switch is added. subnets = "" # A comma delimited string of subnets to place the worker nodes in. i.e. subnet-123,subnet-456,subnet-789 + autoscaling_enabled = false # Sets whether policy and matching tags will be added to allow autoscaling. } } diff --git a/workers.tf b/workers.tf index 070ee86..61258ea 100644 --- a/workers.tf +++ b/workers.tf @@ -5,12 +5,13 @@ resource "aws_autoscaling_group" "workers" { min_size = "${lookup(var.worker_groups[count.index], "asg_min_size",lookup(var.workers_group_defaults, "asg_min_size"))}" launch_configuration = "${element(aws_launch_configuration.workers.*.id, count.index)}" vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups[count.index], "subnets", ""), join(",", var.subnets)))}"] - count = "${var.worker_group_count}" + count = "${length(var.worker_groups)}" tags = ["${concat( list( map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true), map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true), + map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups[count.index], "autoscaling_enabled", count.index) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false), ), local.asg_tags) }"] @@ -32,7 +33,7 @@ resource "aws_launch_configuration" "workers" { ebs_optimized = "${lookup(var.worker_groups[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_groups[count.index], "instance_type", lookup(var.workers_group_defaults, "instance_type")), false))}" enable_monitoring = "${lookup(var.worker_groups[count.index], "enable_monitoring", lookup(var.workers_group_defaults, "enable_monitoring"))}" spot_price = "${lookup(var.worker_groups[count.index], "spot_price", lookup(var.workers_group_defaults, "spot_price"))}" - count = "${var.worker_group_count}" + count = "${length(var.worker_groups)}" lifecycle { create_before_destroy = true @@ -122,3 +123,56 @@ resource "null_resource" "tags_as_list_of_maps" { "propagate_at_launch", "true" )}" } + +resource "aws_iam_role_policy_attachment" "workers_autoscaling" { + policy_arn = "${aws_iam_policy.worker_autoscaling.arn}" + role = "${aws_iam_role.workers.name}" +} + +resource "aws_iam_policy" "worker_autoscaling" { + name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}" + description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" + policy = "${data.aws_iam_policy_document.worker_autoscaling.json}" +} + +data "aws_iam_policy_document" "worker_autoscaling" { + statement { + sid = "eksWorkerAutoscalingAll" + effect = "Allow" + + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "autoscaling:GetAsgForInstance", + ] + + resources = ["*"] + } + + statement { + sid = "eksWorkerAutoscalingOwn" + effect = "Allow" + + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup", + ] + + resources = ["*"] + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${aws_eks_cluster.this.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" + values = ["true"] + } + } +} From 9726fa3e2ac44a2c13afd63f51e55f6c73cf3714 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Mon, 13 Aug 2018 15:12:58 +0200 Subject: [PATCH 2/6] Removing worker_group_count from test fixture --- examples/eks_test_fixture/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/eks_test_fixture/main.tf b/examples/eks_test_fixture/main.tf index b941249..90d115c 100644 --- a/examples/eks_test_fixture/main.tf +++ b/examples/eks_test_fixture/main.tf @@ -73,7 +73,6 @@ module "eks" { tags = "${local.tags}" vpc_id = "${module.vpc.vpc_id}" worker_groups = "${local.worker_groups}" - worker_group_count = "1" map_roles = "${var.map_roles}" map_users = "${var.map_users}" map_accounts = "${var.map_accounts}" From 0f5e36862ee8f986c8da4883132ae073706b3d9c Mon Sep 17 00:00:00 2001 From: Max Williams Date: Mon, 27 Aug 2018 13:45:46 +0200 Subject: [PATCH 3/6] fmt and updating changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f78884..d117db1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added +- Added autoscaling policies into module that are optionally attached when enabled for a worker group. (by @max-rocket-internet) + +### Added + - add spot_price option to aws_launch_configuration - add enable_monitoring option to aws_launch_configuration - add t3 instance class settings From 949c4428dc5a941a0b7e179fbb7e671db5ec6267 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Mon, 27 Aug 2018 15:31:00 +0200 Subject: [PATCH 4/6] reverting calculation of count for worker groups --- README.md | 1 + data.tf | 2 +- variables.tf | 6 ++++++ workers.tf | 4 ++-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 781cd6d..e9f2cc2 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | subnets | A list of subnets to place the EKS cluster and workers within. | list | - | yes | | tags | A map of tags to add to all resources. | map | `` | no | | vpc_id | VPC where the cluster and workers will be deployed. | string | - | yes | +| worker_group_count | The number of maps contained within the worker_groups list. | string | `1` | no | | worker_groups | A list of maps defining worker group configurations. See workers_group_defaults for valid keys. | list | `` | no | | worker_security_group_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `` | no | | worker_sg_ingress_from_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | string | `1025` | no | diff --git a/data.tf b/data.tf index a04e894..3671d1a 100644 --- a/data.tf +++ b/data.tf @@ -71,7 +71,7 @@ EOF data "template_file" "userdata" { template = "${file("${path.module}/templates/userdata.sh.tpl")}" - count = "${length(var.worker_groups)}" + count = "${var.worker_group_count}" vars { region = "${data.aws_region.current.name}" diff --git a/variables.tf b/variables.tf index bc672b6..979d42d 100644 --- a/variables.tf +++ b/variables.tf @@ -69,6 +69,12 @@ variable "worker_groups" { }] } +variable "worker_group_count" { + description = "The number of maps contained within the worker_groups list." + type = "string" + default = "1" +} + variable "workers_group_defaults" { description = "Default values for target groups as defined by the list of maps." type = "map" diff --git a/workers.tf b/workers.tf index 61258ea..77783ff 100644 --- a/workers.tf +++ b/workers.tf @@ -5,7 +5,7 @@ resource "aws_autoscaling_group" "workers" { min_size = "${lookup(var.worker_groups[count.index], "asg_min_size",lookup(var.workers_group_defaults, "asg_min_size"))}" launch_configuration = "${element(aws_launch_configuration.workers.*.id, count.index)}" vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups[count.index], "subnets", ""), join(",", var.subnets)))}"] - count = "${length(var.worker_groups)}" + count = "${var.worker_group_count}" tags = ["${concat( list( @@ -33,7 +33,7 @@ resource "aws_launch_configuration" "workers" { ebs_optimized = "${lookup(var.worker_groups[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_groups[count.index], "instance_type", lookup(var.workers_group_defaults, "instance_type")), false))}" enable_monitoring = "${lookup(var.worker_groups[count.index], "enable_monitoring", lookup(var.workers_group_defaults, "enable_monitoring"))}" spot_price = "${lookup(var.worker_groups[count.index], "spot_price", lookup(var.workers_group_defaults, "spot_price"))}" - count = "${length(var.worker_groups)}" + count = "${var.worker_group_count}" lifecycle { create_before_destroy = true From 2c15e196b70bf49a82dbdc774edc048cb36cd7e2 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Mon, 27 Aug 2018 15:31:00 +0200 Subject: [PATCH 5/6] reverting fixture change also --- examples/eks_test_fixture/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/eks_test_fixture/main.tf b/examples/eks_test_fixture/main.tf index 90d115c..b941249 100644 --- a/examples/eks_test_fixture/main.tf +++ b/examples/eks_test_fixture/main.tf @@ -73,6 +73,7 @@ module "eks" { tags = "${local.tags}" vpc_id = "${module.vpc.vpc_id}" worker_groups = "${local.worker_groups}" + worker_group_count = "1" map_roles = "${var.map_roles}" map_users = "${var.map_users}" map_accounts = "${var.map_accounts}" From c37dc6d879e4317950901db2b9c42119b290cbe4 Mon Sep 17 00:00:00 2001 From: Max Williams Date: Mon, 27 Aug 2018 16:40:46 +0200 Subject: [PATCH 6/6] changelog fix --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d117db1..daea779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,10 @@ project adheres to [Semantic Versioning](http://semver.org/). ### Added -- Added autoscaling policies into module that are optionally attached when enabled for a worker group. (by @max-rocket-internet) - -### Added - - add spot_price option to aws_launch_configuration - add enable_monitoring option to aws_launch_configuration - add t3 instance class settings +- Added autoscaling policies into module that are optionally attached when enabled for a worker group. (by @max-rocket-internet) ### Changed