github-mirrors/terraform-aws-eks
1
0
Fork 0
mirror of https://github.com/ysoftdevs/terraform-aws-eks.git synced 2026-01-16 08:37:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

Adding new mixed type of worker group with instance overrides and mixed instances policy (#371)

Browse Source 
* Adding new mixed type of worker group with instance overrides and mixed instances policy

* moving all count and lifecycle rule parameters to top/bottom

* adding custom IAM parts

* updating doc with new options

* fixes for spot instances
This commit is contained in:
Max Williams
2019-05-07 16:50:42 +02:00
committed by GitHub
parent 2439c25771
commit ae2f8e58db
13 changed files with 285 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@@ -13,9 +13,10 @@ project adheres to [Semantic Versioning](http://semver.org/).
- Added support for custom service linked role for Auto Scaling group (by @voanhduy1512)
- Added support for custom IAM roles for cluster and workers (by @erks)
- Add cluster arn to outputs (by @alexsn)
- Added cluster ARN to outputs (by @alexsn)
- Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet)
- Added doc about spot instances (by @max-rocket-internet)
- Added new worker group option with a mixed instances policy (by @max-rocket-internet)
### Changed

2
README.md
View File

@@ -148,6 +148,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a
| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no |
| worker\_group\_count | The number of maps contained within the worker_groups list. | string | `"1"` | no |
| worker\_group\_launch\_template\_count | The number of maps contained within the worker_groups_launch_template list. | string | `"0"` | no |
| worker\_group\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |
| worker\_group\_launch\_template\_mixed\_count | The number of maps contained within the worker_group_launch_template_mixed list. | string | `"0"` | no |
| worker\_group\_tags | A map defining extra tags to be applied to the worker group ASG. | map | `{ "default": [] }` | no |
| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |
| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |

5
aws_auth.tf
View File

@@ -1,10 +1,11 @@
resource "local_file" "config_map_aws_auth" {
count = "${var.write_aws_auth_config ? 1 : 0}"
content = "${data.template_file.config_map_aws_auth.rendered}"
filename = "${var.config_output_path}config-map-aws-auth_${var.cluster_name}.yaml"
count = "${var.write_aws_auth_config ? 1 : 0}"
}
resource "null_resource" "update_config_map_aws_auth" {
count = "${var.manage_aws_auth ? 1 : 0}"
depends_on = ["aws_eks_cluster.this"]
provisioner "local-exec" {
@@ -28,8 +29,6 @@ EOS
config_map_rendered = "${data.template_file.config_map_aws_auth.rendered}"
endpoint = "${aws_eks_cluster.this.endpoint}"
}
count = "${var.manage_aws_auth ? 1 : 0}"
}
data "aws_caller_identity" "current" {}

12
cluster.tf
View File

@@ -23,14 +23,15 @@ resource "aws_eks_cluster" "this" {
}
resource "aws_security_group" "cluster" {
count = "${var.cluster_create_security_group ? 1 : 0}"
name_prefix = "${var.cluster_name}"
description = "EKS cluster security group."
vpc_id = "${var.vpc_id}"
tags = "${merge(var.tags, map("Name", "${var.cluster_name}-eks_cluster_sg"))}"
count = "${var.cluster_create_security_group ? 1 : 0}"
}
resource "aws_security_group_rule" "cluster_egress_internet" {
count = "${var.cluster_create_security_group ? 1 : 0}"
description = "Allow cluster egress access to the Internet."
protocol = "-1"
security_group_id = "${aws_security_group.cluster.id}"
@@ -38,10 +39,10 @@ resource "aws_security_group_rule" "cluster_egress_internet" {
from_port = 0
to_port = 0
type = "egress"
count = "${var.cluster_create_security_group ? 1 : 0}"
}
resource "aws_security_group_rule" "cluster_https_worker_ingress" {
count = "${var.cluster_create_security_group ? 1 : 0}"
description = "Allow pods to communicate with the EKS cluster API."
protocol = "tcp"
security_group_id = "${aws_security_group.cluster.id}"
@@ -49,26 +50,25 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" {
from_port = 443
to_port = 443
type = "ingress"
count = "${var.cluster_create_security_group ? 1 : 0}"
}
resource "aws_iam_role" "cluster" {
count = "${var.manage_cluster_iam_resources ? 1 : 0}"
name_prefix = "${var.cluster_name}"
assume_role_policy = "${data.aws_iam_policy_document.cluster_assume_role_policy.json}"
permissions_boundary = "${var.permissions_boundary}"
path = "${var.iam_path}"
force_detach_policies = true
count = "${var.manage_cluster_iam_resources ? 1 : 0}"
}
resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" {
count = "${var.manage_cluster_iam_resources ? 1 : 0}"
policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
role = "${aws_iam_role.cluster.name}"
count = "${var.manage_cluster_iam_resources ? 1 : 0}"
}
resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" {
count = "${var.manage_cluster_iam_resources ? 1 : 0}"
policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
role = "${aws_iam_role.cluster.name}"
count = "${var.manage_cluster_iam_resources ? 1 : 0}"
}

34
data.tf
View File

@@ -58,13 +58,13 @@ data "template_file" "kubeconfig" {
}
data "template_file" "aws_authenticator_env_variables" {
count = "${length(var.kubeconfig_aws_authenticator_env_variables)}"
template = <<EOF
- name: $${key}
value: $${value}
EOF
count = "${length(var.kubeconfig_aws_authenticator_env_variables)}"
vars {
value = "${element(values(var.kubeconfig_aws_authenticator_env_variables), count.index)}"
key = "${element(keys(var.kubeconfig_aws_authenticator_env_variables), count.index)}"
@@ -72,8 +72,8 @@ EOF
}
data "template_file" "userdata" {
template = "${file("${path.module}/templates/userdata.sh.tpl")}"
count = "${var.worker_group_count}"
template = "${file("${path.module}/templates/userdata.sh.tpl")}"
vars {
cluster_name = "${aws_eks_cluster.this.name}"
@@ -87,8 +87,23 @@ data "template_file" "userdata" {
}
data "template_file" "launch_template_userdata" {
template = "${file("${path.module}/templates/userdata.sh.tpl")}"
count = "${var.worker_group_launch_template_count}"
template = "${file("${path.module}/templates/userdata.sh.tpl")}"
vars {
cluster_name = "${aws_eks_cluster.this.name}"
endpoint = "${aws_eks_cluster.this.endpoint}"
cluster_auth_base64 = "${aws_eks_cluster.this.certificate_authority.0.data}"
pre_userdata = "${lookup(var.worker_groups_launch_template[count.index], "pre_userdata", local.workers_group_defaults["pre_userdata"])}"
additional_userdata = "${lookup(var.worker_groups_launch_template[count.index], "additional_userdata", local.workers_group_defaults["additional_userdata"])}"
bootstrap_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "bootstrap_extra_args", local.workers_group_defaults["bootstrap_extra_args"])}"
kubelet_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"])}"
}
}
data "template_file" "workers_launch_template_mixed" {
count = "${var.worker_group_launch_template_mixed_count}"
template = "${file("${path.module}/templates/userdata.sh.tpl")}"
vars {
cluster_name = "${aws_eks_cluster.this.name}"
@@ -102,16 +117,21 @@ data "template_file" "launch_template_userdata" {
}
data "aws_iam_role" "custom_cluster_iam_role" {
name = "${var.cluster_iam_role_name}"
count = "${var.manage_cluster_iam_resources ? 0 : 1}"
name = "${var.cluster_iam_role_name}"
}
data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" {
name = "${lookup(var.worker_groups[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_count}"
name = "${lookup(var.worker_groups[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
}
data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" {
name = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_count}"
name = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
}
data "aws_iam_instance_profile" "custom_worker_group_launch_template_mixed_iam_instance_profile" {
count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_mixed_count}"
name = "${lookup(var.worker_group_launch_template_mixed[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
}

123
docs/spot-instances.md
View File

@@ -1,6 +1,6 @@
# Using spot instances
Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they will be terminated with only 2 minutes warning.
Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they could be terminated with only 2 minutes warning.
You need to install a daemonset to catch the 2 minute warning before termination. This will ensure the node is gracefully drained before termination. You can install the [k8s-spot-termination-handler](https://github.com/kube-aws/kube-spot-termination-notice-handler) for this. There's a [Helm chart](https://github.com/helm/charts/tree/master/stable/k8s-spot-termination-handler):
@@ -8,94 +8,91 @@ You need to install a daemonset to catch the 2 minute warning before termination
helm install stable/k8s-spot-termination-handler --namespace kube-system
```
In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler):
In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload not suitable for spot instances but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler):
```yaml
nodeSelector:
spot: "false"
kubernetes.io/lifecycle: spot
```
Notes:
- The `spot_price` is set to the on-demand price so that the spot instances will run as long as they are the cheaper.
- It's best to have a broad range of instance types to ensure there's always some instances to run when prices fluctuate.
- Using an [AWS Spot Fleet](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html) is the best option but is not supported by this module yet.
- There is an AWS blog article about this [here](https://aws.amazon.com/blogs/compute/run-your-kubernetes-workloads-on-amazon-ec2-spot-instances-with-amazon-eks/).
- Consider using [k8s-spot-rescheduler](https://github.com/pusher/k8s-spot-rescheduler) to move pods from on-demand to spot instances.
## Using Launch Configuration
Example Terraform worker group configuration that use an ASG with launch configuration:
Example worker group configuration that uses an ASG with launch configuration for each worker group:
```hcl
worker_group_count = 3
worker_group_count = 3
worker_groups = [
{
name = "on-demand-1"
instance_type = "m4.xlarge"
asg_max_size = 1
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=spot=false"
suspended_processes = "AZRebalance"
},
{
name = "spot-1"
spot_price = "0.39"
instance_type = "c4.2xlarge"
asg_max_size = 20
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=spot=true"
suspended_processes = "AZRebalance"
},
{
name = "spot-2"
spot_price = "0.40"
instance_type = "m4.2xlarge"
asg_max_size = 20
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=spot=true"
suspended_processes = "AZRebalance"
}
]
worker_groups = [
{
name = "on-demand-1"
instance_type = "m4.xlarge"
asg_max_size = 1
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=normal"
suspended_processes = "AZRebalance"
},
{
name = "spot-1"
spot_price = "0.199"
instance_type = "c4.xlarge"
asg_max_size = 20
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot"
suspended_processes = "AZRebalance"
},
{
name = "spot-2"
spot_price = "0.20"
instance_type = "m4.xlarge"
asg_max_size = 20
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot"
suspended_processes = "AZRebalance"
}
]
```
## Using Launch Templates
Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested.
Example Terraform worker group configuration that use an ASG with a launch template:
Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested but it's more suitable for spot instances as it allowed multiple instance types in the same worker group:
```hcl
worker_group_count = 1
worker_group_count = 1
worker_groups = [
{
name = "on-demand-1"
instance_type = "m4.xlarge"
asg_max_size = 10
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=spot=false"
suspended_processes = "AZRebalance"
}
]
worker_groups = [
{
name = "on-demand-1"
instance_type = "m4.xlarge"
asg_max_size = 10
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=spot=false"
suspended_processes = "AZRebalance"
}
]
worker_group_launch_template_mixed_count = 1
worker_group_launch_template_count = 1
worker_groups_launch_template = [
{
name = "spot-1"
instance_type = "m5.xlarge"
override_instance_type = "m4.xlarge"
spot_instance_pools = 2
on_demand_percentage_above_base_capacity = 0
spot_max_price = "0.384"
asg_max_size = 10
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=spot=true"
}
]
worker_group_launch_template_mixed = [
{
name = "spot-1"
override_instance_type_1 = "m5.large"
override_instance_type_2 = "c5.large"
override_instance_type_3 = "t3.large"
override_instance_type_4 = "r5.large"
spot_instance_pools = 3
asg_max_size = 5
asg_desired_size = 5
autoscaling_enabled = true
kubelet_extra_args = "--node-labels=kubernetes.io/lifecycle=spot"
}
]
```
## Important issues

2
kubectl.tf
View File

@@ -1,5 +1,5 @@
resource "local_file" "kubeconfig" {
count = "${var.write_kubeconfig ? 1 : 0}"
content = "${data.template_file.kubeconfig.rendered}"
filename = "${var.config_output_path}kubeconfig_${var.cluster_name}"
count = "${var.write_kubeconfig ? 1 : 0}"
}

12
local.tf
View File

@@ -53,6 +53,18 @@ locals {
launch_template_placement_group = "" # The name of the placement group into which to launch the instances, if any.
root_encrypted = "" # Whether the volume should be encrypted or not
eni_delete = true # Delete the ENI on termination (if set to false you will have to manually delete before destroying)
# Settings for launch templates with mixed instances policy
override_instance_type_1 = "m5.large" # Override instance type 1 for mixed instances policy
override_instance_type_2 = "c5.large" # Override instance type 2 for mixed instances policy
override_instance_type_3 = "t3.large" # Override instance type 3 for mixed instances policy
override_instance_type_4 = "r5.large" # Override instance type 4 for mixed instances policy
on_demand_allocation_strategy = "prioritized" # Strategy to use when launching on-demand instances. Valid values: prioritized.
on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances
on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity
spot_allocation_strategy = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify.
spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify."
spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price
}
workers_group_defaults = "${merge(local.workers_group_defaults_defaults, var.workers_group_defaults)}"

8
outputs.tf
View File

@@ -55,12 +55,12 @@ output "kubeconfig_filename" {
output "workers_asg_arns" {
description = "IDs of the autoscaling groups containing workers."
value = "${concat(aws_autoscaling_group.workers.*.arn, aws_autoscaling_group.workers_launch_template.*.arn)}"
value = "${concat(aws_autoscaling_group.workers.*.arn, aws_autoscaling_group.workers_launch_template.*.arn, aws_autoscaling_group.workers_launch_template_mixed.*.arn)}"
}
output "workers_asg_names" {
description = "Names of the autoscaling groups containing workers."
value = "${concat(aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id)}"
value = "${concat(aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id, aws_autoscaling_group.workers_launch_template_mixed.*.id)}"
}
output "workers_user_data" {
@@ -105,10 +105,10 @@ output "worker_iam_instance_profile_names" {
output "worker_iam_role_name" {
description = "default IAM role name for EKS worker groups"
value = "${element(coalescelist(aws_iam_role.workers.*.name, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name), 0)}"
value = "${element(coalescelist(aws_iam_role.workers.*.name, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_name), 0)}"
}
output "worker_iam_role_arn" {
description = "default IAM role ARN for EKS worker groups"
value = "${element(coalescelist(aws_iam_role.workers.*.arn, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn), 0)}"
value = "${element(coalescelist(aws_iam_role.workers.*.arn, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_arn), 0)}"
}

17
variables.tf
View File

@@ -132,6 +132,23 @@ variable "worker_groups_launch_template" {
]
}
variable "worker_group_launch_template_mixed" {
description = "A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys."
type = "list"
default = [
{
"name" = "default"
},
]
}
variable "worker_group_launch_template_mixed_count" {
description = "The number of maps contained within the worker_group_launch_template_mixed list."
type = "string"
default = "0"
}
variable "worker_group_launch_template_count" {
description = "The number of maps contained within the worker_groups_launch_template list."
type = "string"

44
workers.tf
View File

@@ -1,6 +1,7 @@
# Worker Groups using Launch Configurations
resource "aws_autoscaling_group" "workers" {
count = "${var.worker_group_count}"
name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}"
desired_capacity = "${lookup(var.worker_groups[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}"
max_size = "${lookup(var.worker_groups[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}"
@@ -13,7 +14,6 @@ resource "aws_autoscaling_group" "workers" {
protect_from_scale_in = "${lookup(var.worker_groups[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}"
suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"]
enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"]
count = "${var.worker_group_count}"
placement_group = "${lookup(var.worker_groups[count.index], "placement_group", local.workers_group_defaults["placement_group"])}"
tags = ["${concat(
@@ -30,12 +30,12 @@ resource "aws_autoscaling_group" "workers" {
lifecycle {
create_before_destroy = true
ignore_changes = ["desired_capacity"]
ignore_changes = ["desired_capacity"]
}
}
resource "aws_launch_configuration" "workers" {
count = "${var.worker_group_count}"
name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}"
associate_public_ip_address = "${lookup(var.worker_groups[count.index], "public_ip", local.workers_group_defaults["public_ip"])}"
security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_groups[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"]
@@ -48,11 +48,6 @@ resource "aws_launch_configuration" "workers" {
enable_monitoring = "${lookup(var.worker_groups[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}"
spot_price = "${lookup(var.worker_groups[count.index], "spot_price", local.workers_group_defaults["spot_price"])}"
placement_tenancy = "${lookup(var.worker_groups[count.index], "placement_tenancy", local.workers_group_defaults["placement_tenancy"])}"
count = "${var.worker_group_count}"
lifecycle {
create_before_destroy = true
}
root_block_device {
volume_size = "${lookup(var.worker_groups[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}"
@@ -60,18 +55,23 @@ resource "aws_launch_configuration" "workers" {
iops = "${lookup(var.worker_groups[count.index], "root_iops", local.workers_group_defaults["root_iops"])}"
delete_on_termination = true
}
lifecycle {
create_before_destroy = true
}
}
resource "aws_security_group" "workers" {
count = "${var.worker_create_security_group ? 1 : 0}"
name_prefix = "${aws_eks_cluster.this.name}"
description = "Security group for all nodes in the cluster."
vpc_id = "${var.vpc_id}"
count = "${var.worker_create_security_group ? 1 : 0}"
tags = "${merge(var.tags, map("Name", "${aws_eks_cluster.this.name}-eks_worker_sg", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "owned"
))}"
}
resource "aws_security_group_rule" "workers_egress_internet" {
count = "${var.worker_create_security_group ? 1 : 0}"
description = "Allow nodes all egress to the Internet."
protocol = "-1"
security_group_id = "${aws_security_group.workers.id}"
@@ -79,10 +79,10 @@ resource "aws_security_group_rule" "workers_egress_internet" {
from_port = 0
to_port = 0
type = "egress"
count = "${var.worker_create_security_group ? 1 : 0}"
}
resource "aws_security_group_rule" "workers_ingress_self" {
count = "${var.worker_create_security_group ? 1 : 0}"
description = "Allow node to communicate with each other."
protocol = "-1"
security_group_id = "${aws_security_group.workers.id}"
@@ -90,10 +90,10 @@ resource "aws_security_group_rule" "workers_ingress_self" {
from_port = 0
to_port = 65535
type = "ingress"
count = "${var.worker_create_security_group ? 1 : 0}"
}
resource "aws_security_group_rule" "workers_ingress_cluster" {
count = "${var.worker_create_security_group ? 1 : 0}"
description = "Allow workers pods to receive communication from the cluster control plane."
protocol = "tcp"
security_group_id = "${aws_security_group.workers.id}"
@@ -101,10 +101,10 @@ resource "aws_security_group_rule" "workers_ingress_cluster" {
from_port = "${var.worker_sg_ingress_from_port}"
to_port = 65535
type = "ingress"
count = "${var.worker_create_security_group ? 1 : 0}"
}
resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" {
count = "${var.worker_create_security_group ? (var.worker_sg_ingress_from_port > 10250 ? 1 : 0) : 0}"
description = "Allow workers Kubelets to receive communication from the cluster control plane."
protocol = "tcp"
security_group_id = "${aws_security_group.workers.id}"
@@ -112,10 +112,10 @@ resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" {
from_port = 10250
to_port = 10250
type = "ingress"
count = "${var.worker_create_security_group ? (var.worker_sg_ingress_from_port > 10250 ? 1 : 0) : 0}"
}
resource "aws_security_group_rule" "workers_ingress_cluster_https" {
count = "${var.worker_create_security_group ? 1 : 0}"
description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane."
protocol = "tcp"
security_group_id = "${aws_security_group.workers.id}"
@@ -123,41 +123,41 @@ resource "aws_security_group_rule" "workers_ingress_cluster_https" {
from_port = 443
to_port = 443
type = "ingress"
count = "${var.worker_create_security_group ? 1 : 0}"
}
resource "aws_iam_role" "workers" {
count = "${var.manage_worker_iam_resources ? 1 : 0}"
name_prefix = "${aws_eks_cluster.this.name}"
assume_role_policy = "${data.aws_iam_policy_document.workers_assume_role_policy.json}"
permissions_boundary = "${var.permissions_boundary}"
path = "${var.iam_path}"
force_detach_policies = true
count = "${var.manage_worker_iam_resources ? 1 : 0}"
}
resource "aws_iam_instance_profile" "workers" {
count = "${var.manage_worker_iam_resources ? var.worker_group_count : 0}"
name_prefix = "${aws_eks_cluster.this.name}"
role = "${lookup(var.worker_groups[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}"
count = "${var.manage_worker_iam_resources ? var.worker_group_count : 0}"
path = "${var.iam_path}"
path = "${var.iam_path}"
}
resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" {
count = "${var.manage_worker_iam_resources ? 1 : 0}"
policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
role = "${aws_iam_role.workers.name}"
count = "${var.manage_worker_iam_resources ? 1 : 0}"
}
resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" {
count = "${var.manage_worker_iam_resources ? 1 : 0}"
policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
role = "${aws_iam_role.workers.name}"
count = "${var.manage_worker_iam_resources ? 1 : 0}"
}
resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" {
count = "${var.manage_worker_iam_resources ? 1 : 0}"
policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
role = "${aws_iam_role.workers.name}"
count = "${var.manage_worker_iam_resources ? 1 : 0}"
}
resource "aws_iam_role_policy_attachment" "workers_additional_policies" {
@@ -177,17 +177,17 @@ resource "null_resource" "tags_as_list_of_maps" {
}
resource "aws_iam_role_policy_attachment" "workers_autoscaling" {
count = "${var.manage_worker_iam_resources ? 1 : 0}"
policy_arn = "${aws_iam_policy.worker_autoscaling.arn}"
role = "${aws_iam_role.workers.name}"
count = "${var.manage_worker_iam_resources ? 1 : 0}"
}
resource "aws_iam_policy" "worker_autoscaling" {
count = "${var.manage_worker_iam_resources ? 1 : 0}"
name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}"
description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}"
policy = "${data.aws_iam_policy_document.worker_autoscaling.json}"
path = "${var.iam_path}"
count = "${var.manage_worker_iam_resources ? 1 : 0}"
}
data "aws_iam_policy_document" "worker_autoscaling" {

18
workers_launch_template.tf
View File

@@ -1,6 +1,7 @@
# Worker Groups using Launch Templates
resource "aws_autoscaling_group" "workers_launch_template" {
count = "${var.worker_group_launch_template_count}"
name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}"
desired_capacity = "${lookup(var.worker_groups_launch_template[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}"
max_size = "${lookup(var.worker_groups_launch_template[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}"
@@ -12,7 +13,6 @@ resource "aws_autoscaling_group" "workers_launch_template" {
protect_from_scale_in = "${lookup(var.worker_groups_launch_template[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}"
suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"]
enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"]
count = "${var.worker_group_launch_template_count}"
placement_group = "${lookup(var.worker_groups_launch_template[count.index], "placement_group", local.workers_group_defaults["placement_group"])}"
launch_template {
@@ -34,12 +34,12 @@ resource "aws_autoscaling_group" "workers_launch_template" {
lifecycle {
create_before_destroy = true
ignore_changes = ["desired_capacity"]
ignore_changes = ["desired_capacity"]
}
}
resource "aws_launch_template" "workers_launch_template" {
count = "${var.worker_group_launch_template_count}"
name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}"
network_interfaces {
@@ -72,12 +72,6 @@ resource "aws_launch_template" "workers_launch_template" {
group_name = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}"
}
count = "${var.worker_group_launch_template_count}"
lifecycle {
create_before_destroy = true
}
block_device_mappings {
device_name = "${lookup(var.worker_groups_launch_template[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}"
@@ -92,11 +86,15 @@ resource "aws_launch_template" "workers_launch_template" {
}
tags = "${var.tags}"
lifecycle {
create_before_destroy = true
}
}
resource "aws_iam_instance_profile" "workers_launch_template" {
count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}"
name_prefix = "${aws_eks_cluster.this.name}"
role = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}"
count = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}"
path = "${var.iam_path}"
}

122
workers_launch_template_mixed.tf Normal file
View File

@@ -0,0 +1,122 @@
# Worker Groups using Launch Templates with mixed instances policy
resource "aws_autoscaling_group" "workers_launch_template_mixed" {
count = "${var.worker_group_launch_template_mixed_count}"
name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}"
desired_capacity = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}"
max_size = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}"
min_size = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}"
force_delete = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}"
target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"]
service_linked_role_arn = "${lookup(var.worker_group_launch_template_mixed[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}"
vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"]
protect_from_scale_in = "${lookup(var.worker_group_launch_template_mixed[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}"
suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"]
enabled_metrics = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"]
placement_group = "${lookup(var.worker_group_launch_template_mixed[count.index], "placement_group", local.workers_group_defaults["placement_group"])}"
mixed_instances_policy {
instances_distribution {
on_demand_allocation_strategy = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"])}"
on_demand_base_capacity = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_base_capacity", local.workers_group_defaults["on_demand_base_capacity"])}"
on_demand_percentage_above_base_capacity = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_percentage_above_base_capacity", local.workers_group_defaults["on_demand_percentage_above_base_capacity"])}"
spot_allocation_strategy = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_allocation_strategy", local.workers_group_defaults["spot_allocation_strategy"])}"
spot_instance_pools = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_instance_pools", local.workers_group_defaults["spot_instance_pools"])}"
spot_max_price = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_max_price", local.workers_group_defaults["spot_max_price"])}"
}
launch_template {
launch_template_specification {
launch_template_id = "${element(aws_launch_template.workers_launch_template_mixed.*.id, count.index)}"
version = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_version", local.workers_group_defaults["launch_template_version"])}"
}
override {
instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_1", local.workers_group_defaults["override_instance_type_1"])}"
}
override {
instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_2", local.workers_group_defaults["override_instance_type_2"])}"
}
override {
instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_3", local.workers_group_defaults["override_instance_type_3"])}"
}
override {
instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_4", local.workers_group_defaults["override_instance_type_4"])}"
}
}
}
tags = ["${concat(
list(
map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true),
map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true),
map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_group_launch_template_mixed[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false),
map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false),
map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false)
),
local.asg_tags,
var.worker_group_tags[contains(keys(var.worker_group_tags), "${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}") ? "${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}" : "default"])
}"]
lifecycle {
create_before_destroy = true
ignore_changes = ["desired_capacity"]
}
}
resource "aws_launch_template" "workers_launch_template_mixed" {
count = "${var.worker_group_launch_template_mixed_count}"
name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}"
network_interfaces {
associate_public_ip_address = "${lookup(var.worker_group_launch_template_mixed[count.index], "public_ip", local.workers_group_defaults["public_ip"])}"
delete_on_termination = "${lookup(var.worker_group_launch_template_mixed[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}"
security_groups = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_group_launch_template_mixed[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"]
}
iam_instance_profile {
name = "${element(coalescelist(aws_iam_instance_profile.workers_launch_template_mixed.*.name, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.name), count.index)}"
}
image_id = "${lookup(var.worker_group_launch_template_mixed[count.index], "ami_id", local.workers_group_defaults["ami_id"])}"
instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"])}"
key_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "key_name", local.workers_group_defaults["key_name"])}"
user_data = "${base64encode(element(data.template_file.workers_launch_template_mixed.*.rendered, count.index))}"
ebs_optimized = "${lookup(var.worker_group_launch_template_mixed[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_group_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"]), false))}"
monitoring {
enabled = "${lookup(var.worker_group_launch_template_mixed[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}"
}
placement {
tenancy = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_placement_tenancy", local.workers_group_defaults["launch_template_placement_tenancy"])}"
group_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}"
}
block_device_mappings {
device_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}"
ebs {
volume_size = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}"
volume_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_type", local.workers_group_defaults["root_volume_type"])}"
iops = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_iops", local.workers_group_defaults["root_iops"])}"
encrypted = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_encrypted", local.workers_group_defaults["root_encrypted"])}"
kms_key_id = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_kms_key_id", local.workers_group_defaults["root_kms_key_id"])}"
delete_on_termination = true
}
}
lifecycle {
create_before_destroy = true
}
}
resource "aws_iam_instance_profile" "workers_launch_template_mixed" {
count = "${var.worker_group_launch_template_mixed_count}"
name_prefix = "${aws_eks_cluster.this.name}"
role = "${lookup(var.worker_group_launch_template_mixed[count.index], "iam_role_id", lookup(local.workers_group_defaults, "iam_role_id"))}"
path = "${var.iam_path}"
}