From ae2f8e58dbe1fc1bec1dbd6861a3fd2bc648f548 Mon Sep 17 00:00:00 2001
From: Max Williams <max.williams@deliveryhero.com>
Date: Tue, 7 May 2019 16:50:42 +0200
Subject: [PATCH] Adding new mixed type of worker group with instance overrides
 and mixed instances policy (#371)

* Adding new mixed type of worker group with instance overrides and mixed instances policy

* moving all count and lifecycle rule parameters to top/bottom

* adding custom IAM parts

* updating doc with new options

* fixes for spot instances
---
 CHANGELOG.md                     |   3 +-
 README.md                        |   2 +
 aws_auth.tf                      |   5 +-
 cluster.tf                       |  12 +--
 data.tf                          |  34 +++++++--
 docs/spot-instances.md           | 123 +++++++++++++++----------------
 kubectl.tf                       |   2 +-
 local.tf                         |  12 +++
 outputs.tf                       |   8 +-
 variables.tf                     |  17 +++++
 workers.tf                       |  44 +++++------
 workers_launch_template.tf       |  18 ++---
 workers_launch_template_mixed.tf | 122 ++++++++++++++++++++++++++++++
 13 files changed, 285 insertions(+), 117 deletions(-)
 create mode 100644 workers_launch_template_mixed.tf

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e01d73e..374b54a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,9 +13,10 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512)
 - Added support for custom IAM roles for cluster and workers (by @erks)
-- Add cluster arn to outputs (by @alexsn)
+- Added cluster ARN to outputs (by @alexsn)
 - Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet)
 - Added doc about spot instances (by @max-rocket-internet)
+- Added new worker group option with a mixed instances policy (by @max-rocket-internet)
 
 ### Changed
 
diff --git a/README.md b/README.md
index b1875a3..61dc4d1 100644
--- a/README.md
+++ b/README.md
@@ -148,6 +148,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a
 | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no |
 | worker\_group\_count | The number of maps contained within the worker_groups list. | string | `"1"` | no |
 | worker\_group\_launch\_template\_count | The number of maps contained within the worker_groups_launch_template list. | string | `"0"` | no |
+| worker\_group\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |
+| worker\_group\_launch\_template\_mixed\_count | The number of maps contained within the worker_group_launch_template_mixed list. | string | `"0"` | no |
 | worker\_group\_tags | A map defining extra tags to be applied to the worker group ASG. | map | `{ "default": [] }` | no |
 | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |
 | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |
diff --git a/aws_auth.tf b/aws_auth.tf
index a86349c..a9fc899 100644
--- a/aws_auth.tf
+++ b/aws_auth.tf
@@ -1,10 +1,11 @@
 resource "local_file" "config_map_aws_auth" {
+  count    = "${var.write_aws_auth_config ? 1 : 0}"
   content  = "${data.template_file.config_map_aws_auth.rendered}"
   filename = "${var.config_output_path}config-map-aws-auth_${var.cluster_name}.yaml"
-  count    = "${var.write_aws_auth_config ? 1 : 0}"
 }
 
 resource "null_resource" "update_config_map_aws_auth" {
+  count      = "${var.manage_aws_auth ? 1 : 0}"
   depends_on = ["aws_eks_cluster.this"]
 
   provisioner "local-exec" {
@@ -28,8 +29,6 @@ EOS
     config_map_rendered      = "${data.template_file.config_map_aws_auth.rendered}"
     endpoint                 = "${aws_eks_cluster.this.endpoint}"
   }
-
-  count = "${var.manage_aws_auth ? 1 : 0}"
 }
 
 data "aws_caller_identity" "current" {}
diff --git a/cluster.tf b/cluster.tf
index 5877c62..dc789ee 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -23,14 +23,15 @@ resource "aws_eks_cluster" "this" {
 }
 
 resource "aws_security_group" "cluster" {
+  count       = "${var.cluster_create_security_group ? 1 : 0}"
   name_prefix = "${var.cluster_name}"
   description = "EKS cluster security group."
   vpc_id      = "${var.vpc_id}"
   tags        = "${merge(var.tags, map("Name", "${var.cluster_name}-eks_cluster_sg"))}"
-  count       = "${var.cluster_create_security_group ? 1 : 0}"
 }
 
 resource "aws_security_group_rule" "cluster_egress_internet" {
+  count             = "${var.cluster_create_security_group ? 1 : 0}"
   description       = "Allow cluster egress access to the Internet."
   protocol          = "-1"
   security_group_id = "${aws_security_group.cluster.id}"
@@ -38,10 +39,10 @@ resource "aws_security_group_rule" "cluster_egress_internet" {
   from_port         = 0
   to_port           = 0
   type              = "egress"
-  count             = "${var.cluster_create_security_group ? 1 : 0}"
 }
 
 resource "aws_security_group_rule" "cluster_https_worker_ingress" {
+  count                    = "${var.cluster_create_security_group ? 1 : 0}"
   description              = "Allow pods to communicate with the EKS cluster API."
   protocol                 = "tcp"
   security_group_id        = "${aws_security_group.cluster.id}"
@@ -49,26 +50,25 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" {
   from_port                = 443
   to_port                  = 443
   type                     = "ingress"
-  count                    = "${var.cluster_create_security_group ? 1 : 0}"
 }
 
 resource "aws_iam_role" "cluster" {
+  count                 = "${var.manage_cluster_iam_resources ? 1 : 0}"
   name_prefix           = "${var.cluster_name}"
   assume_role_policy    = "${data.aws_iam_policy_document.cluster_assume_role_policy.json}"
   permissions_boundary  = "${var.permissions_boundary}"
   path                  = "${var.iam_path}"
   force_detach_policies = true
-  count                 = "${var.manage_cluster_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" {
+  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
   role       = "${aws_iam_role.cluster.name}"
-  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" {
+  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
   role       = "${aws_iam_role.cluster.name}"
-  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
 }
diff --git a/data.tf b/data.tf
index 5ef35bd..91e6570 100644
--- a/data.tf
+++ b/data.tf
@@ -58,13 +58,13 @@ data "template_file" "kubeconfig" {
 }
 
 data "template_file" "aws_authenticator_env_variables" {
+  count = "${length(var.kubeconfig_aws_authenticator_env_variables)}"
+
   template = <<EOF
         - name: $${key}
           value: $${value}
 EOF
 
-  count = "${length(var.kubeconfig_aws_authenticator_env_variables)}"
-
   vars {
     value = "${element(values(var.kubeconfig_aws_authenticator_env_variables), count.index)}"
     key   = "${element(keys(var.kubeconfig_aws_authenticator_env_variables), count.index)}"
@@ -72,8 +72,8 @@ EOF
 }
 
 data "template_file" "userdata" {
-  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
   count    = "${var.worker_group_count}"
+  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
 
   vars {
     cluster_name         = "${aws_eks_cluster.this.name}"
@@ -87,8 +87,23 @@ data "template_file" "userdata" {
 }
 
 data "template_file" "launch_template_userdata" {
-  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
   count    = "${var.worker_group_launch_template_count}"
+  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
+
+  vars {
+    cluster_name         = "${aws_eks_cluster.this.name}"
+    endpoint             = "${aws_eks_cluster.this.endpoint}"
+    cluster_auth_base64  = "${aws_eks_cluster.this.certificate_authority.0.data}"
+    pre_userdata         = "${lookup(var.worker_groups_launch_template[count.index], "pre_userdata", local.workers_group_defaults["pre_userdata"])}"
+    additional_userdata  = "${lookup(var.worker_groups_launch_template[count.index], "additional_userdata", local.workers_group_defaults["additional_userdata"])}"
+    bootstrap_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "bootstrap_extra_args", local.workers_group_defaults["bootstrap_extra_args"])}"
+    kubelet_extra_args   = "${lookup(var.worker_groups_launch_template[count.index], "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"])}"
+  }
+}
+
+data "template_file" "workers_launch_template_mixed" {
+  count    = "${var.worker_group_launch_template_mixed_count}"
+  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
 
   vars {
     cluster_name         = "${aws_eks_cluster.this.name}"
@@ -102,16 +117,21 @@ data "template_file" "launch_template_userdata" {
 }
 
 data "aws_iam_role" "custom_cluster_iam_role" {
-  name  = "${var.cluster_iam_role_name}"
   count = "${var.manage_cluster_iam_resources ? 0 : 1}"
+  name  = "${var.cluster_iam_role_name}"
 }
 
 data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" {
-  name  = "${lookup(var.worker_groups[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
   count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_count}"
+  name  = "${lookup(var.worker_groups[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
 }
 
 data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" {
-  name  = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
   count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_count}"
+  name  = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
+}
+
+data "aws_iam_instance_profile" "custom_worker_group_launch_template_mixed_iam_instance_profile" {
+  count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_mixed_count}"
+  name  = "${lookup(var.worker_group_launch_template_mixed[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
 }
diff --git a/docs/spot-instances.md b/docs/spot-instances.md
index 1cc0be1..b476151 100644
--- a/docs/spot-instances.md
+++ b/docs/spot-instances.md
@@ -1,6 +1,6 @@
 # Using spot instances
 
-Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they will be terminated with only 2 minutes warning.
+Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they could be terminated with only 2 minutes warning.
 
 You need to install a daemonset to catch the 2 minute warning before termination. This will ensure the node is gracefully drained before termination. You can install the [k8s-spot-termination-handler](https://github.com/kube-aws/kube-spot-termination-notice-handler) for this. There's a [Helm chart](https://github.com/helm/charts/tree/master/stable/k8s-spot-termination-handler):
 
@@ -8,94 +8,91 @@ You need to install a daemonset to catch the 2 minute warning before termination
 helm install stable/k8s-spot-termination-handler --namespace kube-system
 ```
 
-In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler):
+In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload not suitable for spot instances but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler):
 
 ```yaml
 nodeSelector:
-  spot: "false"
+  kubernetes.io/lifecycle: spot
 ```
 
 Notes:
 
 - The `spot_price` is set to the on-demand price so that the spot instances will run as long as they are the cheaper.
 - It's best to have a broad range of instance types to ensure there's always some instances to run when prices fluctuate.
-- Using an [AWS Spot Fleet](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html) is the best option but is not supported by this module yet.
 - There is an AWS blog article about this [here](https://aws.amazon.com/blogs/compute/run-your-kubernetes-workloads-on-amazon-ec2-spot-instances-with-amazon-eks/).
 - Consider using [k8s-spot-rescheduler](https://github.com/pusher/k8s-spot-rescheduler) to move pods from on-demand to spot instances.
 
 ## Using Launch Configuration
 
-Example Terraform worker group configuration that use an ASG with launch configuration:
+Example worker group configuration that uses an ASG with launch configuration for each worker group:
 
 ```hcl
-worker_group_count = 3
+  worker_group_count = 3
 
-worker_groups = [
-  {
-    name                = "on-demand-1"
-    instance_type       = "m4.xlarge"
-    asg_max_size        = 1
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=false"
-    suspended_processes = "AZRebalance"
-  },
-  {
-    name                = "spot-1"
-    spot_price          = "0.39"
-    instance_type       = "c4.2xlarge"
-    asg_max_size        = 20
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=true"
-    suspended_processes = "AZRebalance"
-  },
-  {
-    name                = "spot-2"
-    spot_price          = "0.40"
-    instance_type       = "m4.2xlarge"
-    asg_max_size        = 20
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=true"
-    suspended_processes = "AZRebalance"
-  }
-]
+  worker_groups = [
+    {
+      name                = "on-demand-1"
+      instance_type       = "m4.xlarge"
+      asg_max_size        = 1
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=kubernetes.io/lifecycle=normal"
+      suspended_processes = "AZRebalance"
+    },
+    {
+      name                = "spot-1"
+      spot_price          = "0.199"
+      instance_type       = "c4.xlarge"
+      asg_max_size        = 20
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=kubernetes.io/lifecycle=spot"
+      suspended_processes = "AZRebalance"
+    },
+    {
+      name                = "spot-2"
+      spot_price          = "0.20"
+      instance_type       = "m4.xlarge"
+      asg_max_size        = 20
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=kubernetes.io/lifecycle=spot"
+      suspended_processes = "AZRebalance"
+    }
+  ]
 ```
 
 ## Using Launch Templates
 
-Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested.
-
-Example Terraform worker group configuration that use an ASG with a launch template:
+Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested but it's more suitable for spot instances as it allowed multiple instance types in the same worker group:
 
 ```hcl
+  worker_group_count = 1
 
-worker_group_count = 1
+  worker_groups = [
+    {
+      name                = "on-demand-1"
+      instance_type       = "m4.xlarge"
+      asg_max_size        = 10
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=spot=false"
+      suspended_processes = "AZRebalance"
+    }
+  ]
 
-worker_groups = [
-  {
-    name                = "on-demand-1"
-    instance_type       = "m4.xlarge"
-    asg_max_size        = 10
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=false"
-    suspended_processes = "AZRebalance"
-  }
-]
+  worker_group_launch_template_mixed_count = 1
 
-worker_group_launch_template_count = 1
-
-worker_groups_launch_template = [
-  {
-    name                                     = "spot-1"
-    instance_type                            = "m5.xlarge"
-    override_instance_type                   = "m4.xlarge"
-    spot_instance_pools                      = 2
-    on_demand_percentage_above_base_capacity = 0
-    spot_max_price                           = "0.384"
-    asg_max_size                             = 10
-    autoscaling_enabled                      = true
-    kubelet_extra_args                       = "--node-labels=spot=true"
-  }
-]
+  worker_group_launch_template_mixed = [
+    {
+      name                     = "spot-1"
+      override_instance_type_1 = "m5.large"
+      override_instance_type_2 = "c5.large"
+      override_instance_type_3 = "t3.large"
+      override_instance_type_4 = "r5.large"
+      spot_instance_pools      = 3
+      asg_max_size             = 5
+      asg_desired_size         = 5
+      autoscaling_enabled      = true
+      kubelet_extra_args       = "--node-labels=kubernetes.io/lifecycle=spot"
+    }
+  ]
 ```
 
 ## Important issues
diff --git a/kubectl.tf b/kubectl.tf
index e39cc8c..7638bb4 100644
--- a/kubectl.tf
+++ b/kubectl.tf
@@ -1,5 +1,5 @@
 resource "local_file" "kubeconfig" {
+  count    = "${var.write_kubeconfig ? 1 : 0}"
   content  = "${data.template_file.kubeconfig.rendered}"
   filename = "${var.config_output_path}kubeconfig_${var.cluster_name}"
-  count    = "${var.write_kubeconfig ? 1 : 0}"
 }
diff --git a/local.tf b/local.tf
index 6efaeba..36ec38f 100644
--- a/local.tf
+++ b/local.tf
@@ -53,6 +53,18 @@ locals {
     launch_template_placement_group   = ""                                            # The name of the placement group into which to launch the instances, if any.
     root_encrypted                    = ""                                            # Whether the volume should be encrypted or not
     eni_delete                        = true                                          # Delete the ENI on termination (if set to false you will have to manually delete before destroying)
+
+    # Settings for launch templates with mixed instances policy
+    override_instance_type_1                 = "m5.large"     # Override instance type 1 for mixed instances policy
+    override_instance_type_2                 = "c5.large"     # Override instance type 2 for mixed instances policy
+    override_instance_type_3                 = "t3.large"     # Override instance type 3 for mixed instances policy
+    override_instance_type_4                 = "r5.large"     # Override instance type 4 for mixed instances policy
+    on_demand_allocation_strategy            = "prioritized"  # Strategy to use when launching on-demand instances. Valid values: prioritized.
+    on_demand_base_capacity                  = "0"            # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances
+    on_demand_percentage_above_base_capacity = "0"            # Percentage split between on-demand and Spot instances above the base on-demand capacity
+    spot_allocation_strategy                 = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify.
+    spot_instance_pools                      = 10             # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify."
+    spot_max_price                           = ""             # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price
   }
 
   workers_group_defaults = "${merge(local.workers_group_defaults_defaults, var.workers_group_defaults)}"
diff --git a/outputs.tf b/outputs.tf
index c99c58d..2db087d 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -55,12 +55,12 @@ output "kubeconfig_filename" {
 
 output "workers_asg_arns" {
   description = "IDs of the autoscaling groups containing workers."
-  value       = "${concat(aws_autoscaling_group.workers.*.arn, aws_autoscaling_group.workers_launch_template.*.arn)}"
+  value       = "${concat(aws_autoscaling_group.workers.*.arn, aws_autoscaling_group.workers_launch_template.*.arn, aws_autoscaling_group.workers_launch_template_mixed.*.arn)}"
 }
 
 output "workers_asg_names" {
   description = "Names of the autoscaling groups containing workers."
-  value       = "${concat(aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id)}"
+  value       = "${concat(aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id, aws_autoscaling_group.workers_launch_template_mixed.*.id)}"
 }
 
 output "workers_user_data" {
@@ -105,10 +105,10 @@ output "worker_iam_instance_profile_names" {
 
 output "worker_iam_role_name" {
   description = "default IAM role name for EKS worker groups"
-  value       = "${element(coalescelist(aws_iam_role.workers.*.name, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name), 0)}"
+  value       = "${element(coalescelist(aws_iam_role.workers.*.name, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_name), 0)}"
 }
 
 output "worker_iam_role_arn" {
   description = "default IAM role ARN for EKS worker groups"
-  value       = "${element(coalescelist(aws_iam_role.workers.*.arn, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn), 0)}"
+  value       = "${element(coalescelist(aws_iam_role.workers.*.arn, data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.role_arn), 0)}"
 }
diff --git a/variables.tf b/variables.tf
index c14f660..c1c4529 100644
--- a/variables.tf
+++ b/variables.tf
@@ -132,6 +132,23 @@ variable "worker_groups_launch_template" {
   ]
 }
 
+variable "worker_group_launch_template_mixed" {
+  description = "A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys."
+  type        = "list"
+
+  default = [
+    {
+      "name" = "default"
+    },
+  ]
+}
+
+variable "worker_group_launch_template_mixed_count" {
+  description = "The number of maps contained within the worker_group_launch_template_mixed list."
+  type        = "string"
+  default     = "0"
+}
+
 variable "worker_group_launch_template_count" {
   description = "The number of maps contained within the worker_groups_launch_template list."
   type        = "string"
diff --git a/workers.tf b/workers.tf
index 75e8580..ce20b1b 100644
--- a/workers.tf
+++ b/workers.tf
@@ -1,6 +1,7 @@
 # Worker Groups using Launch Configurations
 
 resource "aws_autoscaling_group" "workers" {
+  count                   = "${var.worker_group_count}"
   name_prefix             = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}"
   desired_capacity        = "${lookup(var.worker_groups[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}"
   max_size                = "${lookup(var.worker_groups[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}"
@@ -13,7 +14,6 @@ resource "aws_autoscaling_group" "workers" {
   protect_from_scale_in   = "${lookup(var.worker_groups[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}"
   suspended_processes     = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"]
   enabled_metrics         = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"]
-  count                   = "${var.worker_group_count}"
   placement_group         = "${lookup(var.worker_groups[count.index], "placement_group", local.workers_group_defaults["placement_group"])}"
 
   tags = ["${concat(
@@ -30,12 +30,12 @@ resource "aws_autoscaling_group" "workers" {
 
   lifecycle {
     create_before_destroy = true
-
-    ignore_changes = ["desired_capacity"]
+    ignore_changes        = ["desired_capacity"]
   }
 }
 
 resource "aws_launch_configuration" "workers" {
+  count                       = "${var.worker_group_count}"
   name_prefix                 = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}"
   associate_public_ip_address = "${lookup(var.worker_groups[count.index], "public_ip", local.workers_group_defaults["public_ip"])}"
   security_groups             = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_groups[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"]
@@ -48,11 +48,6 @@ resource "aws_launch_configuration" "workers" {
   enable_monitoring           = "${lookup(var.worker_groups[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}"
   spot_price                  = "${lookup(var.worker_groups[count.index], "spot_price", local.workers_group_defaults["spot_price"])}"
   placement_tenancy           = "${lookup(var.worker_groups[count.index], "placement_tenancy", local.workers_group_defaults["placement_tenancy"])}"
-  count                       = "${var.worker_group_count}"
-
-  lifecycle {
-    create_before_destroy = true
-  }
 
   root_block_device {
     volume_size           = "${lookup(var.worker_groups[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}"
@@ -60,18 +55,23 @@ resource "aws_launch_configuration" "workers" {
     iops                  = "${lookup(var.worker_groups[count.index], "root_iops", local.workers_group_defaults["root_iops"])}"
     delete_on_termination = true
   }
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 resource "aws_security_group" "workers" {
+  count       = "${var.worker_create_security_group ? 1 : 0}"
   name_prefix = "${aws_eks_cluster.this.name}"
   description = "Security group for all nodes in the cluster."
   vpc_id      = "${var.vpc_id}"
-  count       = "${var.worker_create_security_group ? 1 : 0}"
   tags        = "${merge(var.tags, map("Name", "${aws_eks_cluster.this.name}-eks_worker_sg", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "owned"
   ))}"
 }
 
 resource "aws_security_group_rule" "workers_egress_internet" {
+  count             = "${var.worker_create_security_group ? 1 : 0}"
   description       = "Allow nodes all egress to the Internet."
   protocol          = "-1"
   security_group_id = "${aws_security_group.workers.id}"
@@ -79,10 +79,10 @@ resource "aws_security_group_rule" "workers_egress_internet" {
   from_port         = 0
   to_port           = 0
   type              = "egress"
-  count             = "${var.worker_create_security_group ? 1 : 0}"
 }
 
 resource "aws_security_group_rule" "workers_ingress_self" {
+  count                    = "${var.worker_create_security_group ? 1 : 0}"
   description              = "Allow node to communicate with each other."
   protocol                 = "-1"
   security_group_id        = "${aws_security_group.workers.id}"
@@ -90,10 +90,10 @@ resource "aws_security_group_rule" "workers_ingress_self" {
   from_port                = 0
   to_port                  = 65535
   type                     = "ingress"
-  count                    = "${var.worker_create_security_group ? 1 : 0}"
 }
 
 resource "aws_security_group_rule" "workers_ingress_cluster" {
+  count                    = "${var.worker_create_security_group ? 1 : 0}"
   description              = "Allow workers pods to receive communication from the cluster control plane."
   protocol                 = "tcp"
   security_group_id        = "${aws_security_group.workers.id}"
@@ -101,10 +101,10 @@ resource "aws_security_group_rule" "workers_ingress_cluster" {
   from_port                = "${var.worker_sg_ingress_from_port}"
   to_port                  = 65535
   type                     = "ingress"
-  count                    = "${var.worker_create_security_group ? 1 : 0}"
 }
 
 resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" {
+  count                    = "${var.worker_create_security_group ? (var.worker_sg_ingress_from_port > 10250 ? 1 : 0) : 0}"
   description              = "Allow workers Kubelets to receive communication from the cluster control plane."
   protocol                 = "tcp"
   security_group_id        = "${aws_security_group.workers.id}"
@@ -112,10 +112,10 @@ resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" {
   from_port                = 10250
   to_port                  = 10250
   type                     = "ingress"
-  count                    = "${var.worker_create_security_group ? (var.worker_sg_ingress_from_port > 10250 ? 1 : 0) : 0}"
 }
 
 resource "aws_security_group_rule" "workers_ingress_cluster_https" {
+  count                    = "${var.worker_create_security_group ? 1 : 0}"
   description              = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane."
   protocol                 = "tcp"
   security_group_id        = "${aws_security_group.workers.id}"
@@ -123,41 +123,41 @@ resource "aws_security_group_rule" "workers_ingress_cluster_https" {
   from_port                = 443
   to_port                  = 443
   type                     = "ingress"
-  count                    = "${var.worker_create_security_group ? 1 : 0}"
 }
 
 resource "aws_iam_role" "workers" {
+  count                 = "${var.manage_worker_iam_resources ? 1 : 0}"
   name_prefix           = "${aws_eks_cluster.this.name}"
   assume_role_policy    = "${data.aws_iam_policy_document.workers_assume_role_policy.json}"
   permissions_boundary  = "${var.permissions_boundary}"
   path                  = "${var.iam_path}"
   force_detach_policies = true
-  count                 = "${var.manage_worker_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_instance_profile" "workers" {
+  count       = "${var.manage_worker_iam_resources ? var.worker_group_count : 0}"
   name_prefix = "${aws_eks_cluster.this.name}"
   role        = "${lookup(var.worker_groups[count.index], "iam_role_id",  lookup(local.workers_group_defaults, "iam_role_id"))}"
-  count       = "${var.manage_worker_iam_resources ? var.worker_group_count : 0}"
-  path        = "${var.iam_path}"
+
+  path = "${var.iam_path}"
 }
 
 resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" {
+  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
   role       = "${aws_iam_role.workers.name}"
-  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" {
+  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
   role       = "${aws_iam_role.workers.name}"
-  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" {
+  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
   role       = "${aws_iam_role.workers.name}"
-  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_role_policy_attachment" "workers_additional_policies" {
@@ -177,17 +177,17 @@ resource "null_resource" "tags_as_list_of_maps" {
 }
 
 resource "aws_iam_role_policy_attachment" "workers_autoscaling" {
+  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
   policy_arn = "${aws_iam_policy.worker_autoscaling.arn}"
   role       = "${aws_iam_role.workers.name}"
-  count      = "${var.manage_worker_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_policy" "worker_autoscaling" {
+  count       = "${var.manage_worker_iam_resources ? 1 : 0}"
   name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}"
   description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}"
   policy      = "${data.aws_iam_policy_document.worker_autoscaling.json}"
   path        = "${var.iam_path}"
-  count       = "${var.manage_worker_iam_resources ? 1 : 0}"
 }
 
 data "aws_iam_policy_document" "worker_autoscaling" {
diff --git a/workers_launch_template.tf b/workers_launch_template.tf
index 99c26dd..efdafb1 100644
--- a/workers_launch_template.tf
+++ b/workers_launch_template.tf
@@ -1,6 +1,7 @@
 # Worker Groups using Launch Templates
 
 resource "aws_autoscaling_group" "workers_launch_template" {
+  count                   = "${var.worker_group_launch_template_count}"
   name_prefix             = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}"
   desired_capacity        = "${lookup(var.worker_groups_launch_template[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}"
   max_size                = "${lookup(var.worker_groups_launch_template[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}"
@@ -12,7 +13,6 @@ resource "aws_autoscaling_group" "workers_launch_template" {
   protect_from_scale_in   = "${lookup(var.worker_groups_launch_template[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}"
   suspended_processes     = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"]
   enabled_metrics         = ["${compact(split(",", coalesce(lookup(var.worker_groups_launch_template[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"]
-  count                   = "${var.worker_group_launch_template_count}"
   placement_group         = "${lookup(var.worker_groups_launch_template[count.index], "placement_group", local.workers_group_defaults["placement_group"])}"
 
   launch_template {
@@ -34,12 +34,12 @@ resource "aws_autoscaling_group" "workers_launch_template" {
 
   lifecycle {
     create_before_destroy = true
-
-    ignore_changes = ["desired_capacity"]
+    ignore_changes        = ["desired_capacity"]
   }
 }
 
 resource "aws_launch_template" "workers_launch_template" {
+  count       = "${var.worker_group_launch_template_count}"
   name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups_launch_template[count.index], "name", count.index)}"
 
   network_interfaces {
@@ -72,12 +72,6 @@ resource "aws_launch_template" "workers_launch_template" {
     group_name = "${lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}"
   }
 
-  count = "${var.worker_group_launch_template_count}"
-
-  lifecycle {
-    create_before_destroy = true
-  }
-
   block_device_mappings {
     device_name = "${lookup(var.worker_groups_launch_template[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}"
 
@@ -92,11 +86,15 @@ resource "aws_launch_template" "workers_launch_template" {
   }
 
   tags = "${var.tags}"
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 resource "aws_iam_instance_profile" "workers_launch_template" {
+  count       = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}"
   name_prefix = "${aws_eks_cluster.this.name}"
   role        = "${lookup(var.worker_groups_launch_template[count.index], "iam_role_id",  lookup(local.workers_group_defaults, "iam_role_id"))}"
-  count       = "${var.manage_worker_iam_resources ? var.worker_group_launch_template_count : 0}"
   path        = "${var.iam_path}"
 }
diff --git a/workers_launch_template_mixed.tf b/workers_launch_template_mixed.tf
new file mode 100644
index 0000000..bac40b9
--- /dev/null
+++ b/workers_launch_template_mixed.tf
@@ -0,0 +1,122 @@
+# Worker Groups using Launch Templates with mixed instances policy
+
+resource "aws_autoscaling_group" "workers_launch_template_mixed" {
+  count                   = "${var.worker_group_launch_template_mixed_count}"
+  name_prefix             = "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}"
+  desired_capacity        = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}"
+  max_size                = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}"
+  min_size                = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}"
+  force_delete            = "${lookup(var.worker_group_launch_template_mixed[count.index], "asg_force_delete", local.workers_group_defaults["asg_force_delete"])}"
+  target_group_arns       = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"]
+  service_linked_role_arn = "${lookup(var.worker_group_launch_template_mixed[count.index], "service_linked_role_arn", local.workers_group_defaults["service_linked_role_arn"])}"
+  vpc_zone_identifier     = ["${split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"]
+  protect_from_scale_in   = "${lookup(var.worker_group_launch_template_mixed[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}"
+  suspended_processes     = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"]
+  enabled_metrics         = ["${compact(split(",", coalesce(lookup(var.worker_group_launch_template_mixed[count.index], "enabled_metrics", ""), local.workers_group_defaults["enabled_metrics"])))}"]
+  placement_group         = "${lookup(var.worker_group_launch_template_mixed[count.index], "placement_group", local.workers_group_defaults["placement_group"])}"
+
+  mixed_instances_policy {
+    instances_distribution {
+      on_demand_allocation_strategy            = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"])}"
+      on_demand_base_capacity                  = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_base_capacity", local.workers_group_defaults["on_demand_base_capacity"])}"
+      on_demand_percentage_above_base_capacity = "${lookup(var.worker_group_launch_template_mixed[count.index], "on_demand_percentage_above_base_capacity", local.workers_group_defaults["on_demand_percentage_above_base_capacity"])}"
+      spot_allocation_strategy                 = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_allocation_strategy", local.workers_group_defaults["spot_allocation_strategy"])}"
+      spot_instance_pools                      = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_instance_pools", local.workers_group_defaults["spot_instance_pools"])}"
+      spot_max_price                           = "${lookup(var.worker_group_launch_template_mixed[count.index], "spot_max_price", local.workers_group_defaults["spot_max_price"])}"
+    }
+
+    launch_template {
+      launch_template_specification {
+        launch_template_id = "${element(aws_launch_template.workers_launch_template_mixed.*.id, count.index)}"
+        version            = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_version", local.workers_group_defaults["launch_template_version"])}"
+      }
+
+      override {
+        instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_1", local.workers_group_defaults["override_instance_type_1"])}"
+      }
+
+      override {
+        instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_2", local.workers_group_defaults["override_instance_type_2"])}"
+      }
+
+      override {
+        instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_3", local.workers_group_defaults["override_instance_type_3"])}"
+      }
+
+      override {
+        instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "override_instance_type_4", local.workers_group_defaults["override_instance_type_4"])}"
+      }
+    }
+  }
+
+  tags = ["${concat(
+    list(
+      map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true),
+      map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true),
+      map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_group_launch_template_mixed[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled"  }", "value", "true", "propagate_at_launch", false),
+      map("key", "k8s.io/cluster-autoscaler/${aws_eks_cluster.this.name}", "value", "", "propagate_at_launch", false),
+      map("key", "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage", "value", "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}Gi", "propagate_at_launch", false)
+    ),
+    local.asg_tags,
+    var.worker_group_tags[contains(keys(var.worker_group_tags), "${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}") ? "${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}" : "default"])
+  }"]
+
+  lifecycle {
+    create_before_destroy = true
+    ignore_changes        = ["desired_capacity"]
+  }
+}
+
+resource "aws_launch_template" "workers_launch_template_mixed" {
+  count       = "${var.worker_group_launch_template_mixed_count}"
+  name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_group_launch_template_mixed[count.index], "name", count.index)}"
+
+  network_interfaces {
+    associate_public_ip_address = "${lookup(var.worker_group_launch_template_mixed[count.index], "public_ip", local.workers_group_defaults["public_ip"])}"
+    delete_on_termination       = "${lookup(var.worker_group_launch_template_mixed[count.index], "eni_delete", local.workers_group_defaults["eni_delete"])}"
+    security_groups             = ["${local.worker_security_group_id}", "${var.worker_additional_security_group_ids}", "${compact(split(",",lookup(var.worker_group_launch_template_mixed[count.index],"additional_security_group_ids", local.workers_group_defaults["additional_security_group_ids"])))}"]
+  }
+
+  iam_instance_profile {
+    name = "${element(coalescelist(aws_iam_instance_profile.workers_launch_template_mixed.*.name, data.aws_iam_instance_profile.custom_worker_group_launch_template_mixed_iam_instance_profile.*.name), count.index)}"
+  }
+
+  image_id      = "${lookup(var.worker_group_launch_template_mixed[count.index], "ami_id", local.workers_group_defaults["ami_id"])}"
+  instance_type = "${lookup(var.worker_group_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"])}"
+  key_name      = "${lookup(var.worker_group_launch_template_mixed[count.index], "key_name", local.workers_group_defaults["key_name"])}"
+  user_data     = "${base64encode(element(data.template_file.workers_launch_template_mixed.*.rendered, count.index))}"
+  ebs_optimized = "${lookup(var.worker_group_launch_template_mixed[count.index], "ebs_optimized", lookup(local.ebs_optimized, lookup(var.worker_group_launch_template_mixed[count.index], "instance_type", local.workers_group_defaults["instance_type"]), false))}"
+
+  monitoring {
+    enabled = "${lookup(var.worker_group_launch_template_mixed[count.index], "enable_monitoring", local.workers_group_defaults["enable_monitoring"])}"
+  }
+
+  placement {
+    tenancy    = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_placement_tenancy", local.workers_group_defaults["launch_template_placement_tenancy"])}"
+    group_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])}"
+  }
+
+  block_device_mappings {
+    device_name = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_block_device_name", local.workers_group_defaults["root_block_device_name"])}"
+
+    ebs {
+      volume_size           = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_size", local.workers_group_defaults["root_volume_size"])}"
+      volume_type           = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_volume_type", local.workers_group_defaults["root_volume_type"])}"
+      iops                  = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_iops", local.workers_group_defaults["root_iops"])}"
+      encrypted             = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_encrypted", local.workers_group_defaults["root_encrypted"])}"
+      kms_key_id            = "${lookup(var.worker_group_launch_template_mixed[count.index], "root_kms_key_id", local.workers_group_defaults["root_kms_key_id"])}"
+      delete_on_termination = true
+    }
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_iam_instance_profile" "workers_launch_template_mixed" {
+  count       = "${var.worker_group_launch_template_mixed_count}"
+  name_prefix = "${aws_eks_cluster.this.name}"
+  role        = "${lookup(var.worker_group_launch_template_mixed[count.index], "iam_role_id",  lookup(local.workers_group_defaults, "iam_role_id"))}"
+  path        = "${var.iam_path}"
+}