diff --git a/.gitignore b/.gitignore index b6d358f..e787fbf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ Gemfile.lock terraform.tfstate.d/ kubeconfig config-map-aws-auth.yaml +eks-admin-cluster-role-binding.yaml +eks-admin-service-account.yaml diff --git a/.travis.yml b/.travis.yml index 3c9b98e..b596e57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,21 @@ language: ruby sudo: required dist: trusty + services: - docker + rvm: - 2.4.2 + before_install: - echo "before_install" + install: - echo "install" - gem install bundler --no-rdoc --no-ri - bundle install + before_script: - echo 'before_script' - export AWS_REGION='us-east-1' @@ -22,12 +27,13 @@ before_script: - unzip terraform.zip ; rm -f terraform.zip; chmod +x terraform - mkdir -p ${HOME}/bin ; export PATH=${PATH}:${HOME}/bin; mv terraform ${HOME}/bin/ - terraform -v + script: - echo 'script' - terraform init - terraform fmt -check=true - terraform validate -var "region=${AWS_REGION}" -var "vpc_id=vpc-123456" -var "subnets=[\"subnet-12345a\"]" -var "workers_ami_id=ami-123456" -var "cluster_ingress_cidrs=[]" -var "cluster_name=test_cluster" -- docker run --rm -v $(pwd):/app/ --workdir=/app/ -t wata727/tflint --error-with-issues +# - docker run --rm -v $(pwd):/app/ --workdir=/app/ -t wata727/tflint --error-with-issues - cd examples/eks_test_fixture - terraform init - terraform fmt -check=true @@ -40,6 +46,7 @@ script: # script: ci/deploy.sh # on: # branch: master + notifications: email: recipients: diff --git a/CHANGELOG.md b/CHANGELOG.md index 208e41b..259c420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,22 +5,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [[v1.0.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.2.0...v1.0.0)] - 2018-06-11] + +### Added + +- security group id can be provided for either/both of the cluster and the workers. If not provided, security groups will be created with sufficient rules to allow cluster-worker communication. - kudos to @tanmng on the idea ⭐ +- outputs of security group ids and worker ASG arns added for working with these resources outside the module. + +### Changed + +- Worker build out refactored to allow multiple autoscaling groups each having differing specs. If none are given, a single ASG is created with a set of sane defaults - big thanks to @kppullin 🥨 + ## [[v0.2.0](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.1...v0.2.0)] - 2018-06-08] +### Added + +- ability to specify extra userdata code to execute following kubelet services start. +- EBS optimization used whenever possible for the given instance type. +- When `configure_kubectl_session` is set to true the current shell will be configured to talk to the kubernetes cluster using config files output from the module. + ### Changed - files rendered from dedicated templates to separate out raw code and config from `hcl` - `workers_ami_id` is now made optional. If not specified, the module will source the latest AWS supported EKS AMI instead. -- added ability to specify extra userdata code to execute after the second to configure and start kube services. -- When `configure_kubectl_session` is set to true the current shell will be configured to talk to the kubernetes cluster using config files output from the module. -- EBS optimization used whenever possible for the given instance type. ## [[v0.1.1](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v0.1.0...v0.1.1)] - 2018-06-07] ### Changed -- pre-commit hooks fixed and working. -- made progress on CI, advancing the build to the final `kitchen test` stage before failing. +- Pre-commit hooks fixed and working. +- Made progress on CI, advancing the build to the final `kitchen test` stage before failing. ## [v0.1.0] - 2018-06-07 diff --git a/README.md b/README.md index 395f8dc..9471e63 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,7 @@ A terraform module to create a managed Kubernetes cluster on AWS EKS. Available through the [Terraform registry](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws). Inspired by and adapted from [this doc](https://www.terraform.io/docs/providers/aws/guides/eks-getting-started.html) and its [source code](https://github.com/terraform-providers/terraform-provider-aws/tree/master/examples/eks-getting-started). -Instructions on [this post](https://aws.amazon.com/blogs/aws/amazon-eks-now-generally-available/) -can help guide you through connecting to the cluster via `kubectl`. +Read the [AWS docs on EKS to get connected to the k8s dashboard](https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html). | Branch | Build status | | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -13,8 +12,9 @@ can help guide you through connecting to the cluster via `kubectl`. ## Assumptions -* You want to create a set of resources around an EKS cluster: namely an autoscaling group of workers and a security group for them. -* You've created a Virtual Private Cloud (VPC) and subnets where you intend to put this EKS. +* You want to create an EKS cluster and an autoscaling group of workers for the cluster. +* You want these resources to exist within security groups that allow communication and coordination. These can be user provided or created within the module. +* You've created a Virtual Private Cloud (VPC) and subnets where you intend to put the EKS resources. ## Usage example @@ -28,7 +28,6 @@ module "eks" { subnets = ["subnet-abcde012", "subnet-bcde012a"] tags = "${map("Environment", "test")}" vpc_id = "vpc-abcde012" - cluster_ingress_cidrs = ["24.18.23.91/32"] } ``` @@ -52,8 +51,10 @@ This module has been packaged with [awspec](https://github.com/k1LoW/awspec) tes 3. Ensure your AWS environment is configured (i.e. credentials and region) for test. 4. Test using `bundle exec kitchen test` from the root of the repo. -For now, connectivity to the kubernetes cluster is not tested but will be in the future. -To test your kubectl connection manually, see the [eks_test_fixture README](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_test_fixture/README.md). +For now, connectivity to the kubernetes cluster is not tested but will be in the +future. If `configure_kubectl_session` is set `true`, once the test fixture has +converged, you can query the test cluster from that terminal session with +`kubectl get nodes --watch --kubeconfig kubeconfig`. ## Doc generation @@ -93,30 +94,28 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| additional_userdata | Extra lines of userdata (bash) which are appended to the default userdata code. | string | `` | no | -| cluster_ingress_cidrs | The CIDRs from which we can execute kubectl commands. | list | - | yes | -| cluster_name | Name of the EKS cluster which is also used as a prefix in names of related resources. | string | - | yes | -| cluster_version | Kubernetes version to use for the cluster. | string | `1.10` | no | +| cluster_name | Name of the EKS cluster. Also used as a prefix in names of related resources. | string | - | yes | +| cluster_security_group_id | If provided, the EKS cluster will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the workers and provide API access to your current IP/32. | string | `` | no | +| cluster_version | Kubernetes version to use for the EKS cluster. | string | `1.10` | no | | config_output_path | Determines where config files are placed if using configure_kubectl_session and you want config files to land outside the current working directory. | string | `./` | no | -| configure_kubectl_session | Configure the current session's kubectl to use the instantiated cluster. | string | `false` | no | -| ebs_optimized_workers | If left at default of true, will use ebs optimization if available on the given instance type. | string | `true` | no | -| subnets | A list of subnets to associate with the cluster's underlying instances. | list | - | yes | +| configure_kubectl_session | Configure the current session's kubectl to use the instantiated EKS cluster. | string | `true` | no | +| subnets | A list of subnets to place the EKS cluster and workers within. | list | - | yes | | tags | A map of tags to add to all resources. | string | `` | no | -| vpc_id | VPC id where the cluster and other resources will be deployed. | string | - | yes | -| workers_ami_id | AMI ID for the eks workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI. | string | `` | no | -| workers_asg_desired_capacity | Desired worker capacity in the autoscaling group. | string | `1` | no | -| workers_asg_max_size | Maximum worker capacity in the autoscaling group. | string | `3` | no | -| workers_asg_min_size | Minimum worker capacity in the autoscaling group. | string | `1` | no | -| workers_instance_type | Size of the workers instances. | string | `m4.large` | no | +| vpc_id | VPC where the cluster and workers will be deployed. | string | - | yes | +| worker_groups | A list of maps defining worker group configurations. See workers_group_defaults for valid keys. | list | `` | no | +| worker_security_group_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingres/egress to work with the EKS cluster. | string | `` | no | +| workers_group_defaults | Default values for target groups as defined by the list of maps. | map | `` | no | ## Outputs | Name | Description | |------|-------------| -| cluster_certificate_authority_data | Nested attribute containing certificate-authority-data for your cluster. Tis is the base64 encoded certificate data required to communicate with your cluster. | -| cluster_endpoint | The endpoint for your Kubernetes API server. | -| cluster_id | The name/id of the cluster. | -| cluster_security_group_ids | description | -| cluster_version | The Kubernetes server version for the cluster. | -| config_map_aws_auth | A kubernetes configuration to authenticate to this cluster. | -| kubeconfig | kubectl config file contents for this cluster. | +| cluster_certificate_authority_data | Nested attribute containing certificate-authority-data for your cluster. This is the base64 encoded certificate data required to communicate with your cluster. | +| cluster_endpoint | The endpoint for your EKS Kubernetes API. | +| cluster_id | The name/id of the EKS cluster. | +| cluster_security_group_id | Security group ID attached to the EKS cluster. | +| cluster_version | The Kubernetes server version for the EKS cluster. | +| config_map_aws_auth | A kubernetes configuration to authenticate to this EKS cluster. | +| kubeconfig | kubectl config file contents for this EKS cluster. | +| worker_security_group_id | Security group ID attached to the EKS workers. | +| workers_asg_arns | IDs of the autoscaling groups containing workers. | diff --git a/cluster.tf b/cluster.tf index 1a0fc81..03fe7aa 100644 --- a/cluster.tf +++ b/cluster.tf @@ -4,7 +4,7 @@ resource "aws_eks_cluster" "this" { version = "${var.cluster_version}" vpc_config { - security_group_ids = ["${aws_security_group.cluster.id}"] + security_group_ids = ["${local.cluster_security_group_id}"] subnet_ids = ["${var.subnets}"] } @@ -16,39 +16,43 @@ resource "aws_eks_cluster" "this" { resource "aws_security_group" "cluster" { name_prefix = "${var.cluster_name}" - description = "Cluster communication with workers nodes" + description = "EKS cluster security group." vpc_id = "${var.vpc_id}" tags = "${merge(var.tags, map("Name", "${var.cluster_name}-eks_cluster_sg"))}" + count = "${var.cluster_security_group_id == "" ? 1 : 0}" } resource "aws_security_group_rule" "cluster_egress_internet" { - description = "Allow cluster egress to the Internet." + description = "Allow cluster egress access to the Internet." protocol = "-1" security_group_id = "${aws_security_group.cluster.id}" cidr_blocks = ["0.0.0.0/0"] from_port = 0 to_port = 0 type = "egress" + count = "${var.cluster_security_group_id == "" ? 1 : 0}" } resource "aws_security_group_rule" "cluster_https_worker_ingress" { - description = "Allow pods to communicate with the cluster API Server." + description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = "${aws_security_group.cluster.id}" - source_security_group_id = "${aws_security_group.workers.id}" + source_security_group_id = "${local.worker_security_group_id}" from_port = 443 to_port = 443 type = "ingress" + count = "${var.cluster_security_group_id == "" ? 1 : 0}" } resource "aws_security_group_rule" "cluster_https_cidr_ingress" { - cidr_blocks = ["${var.cluster_ingress_cidrs}"] - description = "Allow communication with the cluster API Server." + cidr_blocks = ["${local.workstation_external_cidr}"] + description = "Allow kubectl communication with the EKS cluster API." protocol = "tcp" security_group_id = "${aws_security_group.cluster.id}" from_port = 443 to_port = 443 type = "ingress" + count = "${var.cluster_security_group_id == "" ? 1 : 0}" } resource "aws_iam_role" "cluster" { diff --git a/data.tf b/data.tf index c25327c..2f08972 100644 --- a/data.tf +++ b/data.tf @@ -1,13 +1,7 @@ data "aws_region" "current" {} -data "aws_ami" "eks_worker" { - filter { - name = "name" - values = ["eks-worker-*"] - } - - most_recent = true - owners = ["602401143452"] # Amazon +data "http" "workstation_external_ip" { + url = "http://icanhazip.com" } data "aws_iam_policy_document" "workers_assume_role_policy" { @@ -25,6 +19,16 @@ data "aws_iam_policy_document" "workers_assume_role_policy" { } } +data "aws_ami" "eks_worker" { + filter { + name = "name" + values = ["eks-worker-*"] + } + + most_recent = true + owners = ["602401143452"] # Amazon +} + data "aws_iam_policy_document" "cluster_assume_role_policy" { statement { sid = "EKSClusterAssumeRole" @@ -40,19 +44,6 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } } -data template_file userdata { - template = "${file("${path.module}/templates/userdata.sh.tpl")}" - - vars { - region = "${data.aws_region.current.name}" - max_pod_count = "${lookup(local.max_pod_per_node, var.workers_instance_type)}" - cluster_name = "${var.cluster_name}" - endpoint = "${aws_eks_cluster.this.endpoint}" - cluster_auth_base64 = "${aws_eks_cluster.this.certificate_authority.0.data}" - additional_userdata = "${var.additional_userdata}" - } -} - data template_file kubeconfig { template = "${file("${path.module}/templates/kubeconfig.tpl")}" @@ -72,7 +63,16 @@ data template_file config_map_aws_auth { } } -module "ebs_optimized" { - source = "./modules/tf_util_ebs_optimized" - instance_type = "${var.workers_instance_type}" +data template_file userdata { + template = "${file("${path.module}/templates/userdata.sh.tpl")}" + count = "${length(var.worker_groups)}" + + vars { + region = "${data.aws_region.current.name}" + cluster_name = "${var.cluster_name}" + endpoint = "${aws_eks_cluster.this.endpoint}" + cluster_auth_base64 = "${aws_eks_cluster.this.certificate_authority.0.data}" + max_pod_count = "${lookup(local.max_pod_per_node, lookup(var.worker_groups[count.index], "instance_type", lookup(var.workers_group_defaults, "instance_type")))}" + additional_userdata = "${lookup(var.worker_groups[count.index], "additional_userdata",lookup(var.workers_group_defaults, "additional_userdata"))}" + } } diff --git a/examples/eks_test_fixture/main.tf b/examples/eks_test_fixture/main.tf index 21687be..459c1ee 100644 --- a/examples/eks_test_fixture/main.tf +++ b/examples/eks_test_fixture/main.tf @@ -11,18 +11,16 @@ provider "random" { version = "= 1.3.1" } -provider "http" {} -provider "local" {} - data "aws_availability_zones" "available" {} -data "http" "workstation_external_ip" { - url = "http://icanhazip.com" -} - locals { - workstation_external_cidr = "${chomp(data.http.workstation_external_ip.body)}/32" - cluster_name = "test-eks-${random_string.suffix.result}" + cluster_name = "test-eks-${random_string.suffix.result}" + + worker_groups = "${list( + map("instance_type","t2.small", + "additional_userdata","echo foo bar" + ), + )}" tags = "${map("Environment", "test", "GithubRepo", "terraform-aws-eks", @@ -50,13 +48,10 @@ module "vpc" { } module "eks" { - source = "../.." - cluster_name = "${local.cluster_name}" - subnets = "${module.vpc.public_subnets}" - tags = "${local.tags}" - vpc_id = "${module.vpc.vpc_id}" - cluster_ingress_cidrs = ["${local.workstation_external_cidr}"] - workers_instance_type = "t2.small" - additional_userdata = "echo hello world" - configure_kubectl_session = true + source = "../.." + cluster_name = "${local.cluster_name}" + subnets = "${module.vpc.public_subnets}" + tags = "${local.tags}" + vpc_id = "${module.vpc.vpc_id}" + worker_groups = "${local.worker_groups}" } diff --git a/examples/eks_test_fixture/outputs.tf b/examples/eks_test_fixture/outputs.tf index 7a656a7..0422d74 100644 --- a/examples/eks_test_fixture/outputs.tf +++ b/examples/eks_test_fixture/outputs.tf @@ -3,9 +3,9 @@ output "cluster_endpoint" { value = "${module.eks.cluster_endpoint}" } -output "cluster_security_group_ids" { +output "cluster_security_group_id" { description = "Security group ids attached to the cluster control plane." - value = "${module.eks.cluster_security_group_ids}" + value = "${module.eks.cluster_security_group_id}" } output "kubectl_config" { diff --git a/kubectl.tf b/kubectl.tf new file mode 100644 index 0000000..e2508f0 --- /dev/null +++ b/kubectl.tf @@ -0,0 +1,24 @@ +resource "local_file" "kubeconfig" { + content = "${data.template_file.kubeconfig.rendered}" + filename = "${var.config_output_path}/kubeconfig" + count = "${var.configure_kubectl_session ? 1 : 0}" +} + +resource "local_file" "config_map_aws_auth" { + content = "${data.template_file.config_map_aws_auth.rendered}" + filename = "${var.config_output_path}/config-map-aws-auth.yaml" + count = "${var.configure_kubectl_session ? 1 : 0}" +} + +resource "null_resource" "configure_kubectl" { + provisioner "local-exec" { + command = "kubectl apply -f ${var.config_output_path}/config-map-aws-auth.yaml --kubeconfig ${var.config_output_path}/kubeconfig" + } + + triggers { + config_map_rendered = "${data.template_file.config_map_aws_auth.rendered}" + kubeconfig_rendered = "${data.template_file.kubeconfig.rendered}" + } + + count = "${var.configure_kubectl_session ? 1 : 0}" +} diff --git a/local.tf b/local.tf index d66ba13..6c3249d 100644 --- a/local.tf +++ b/local.tf @@ -1,4 +1,9 @@ locals { + asg_tags = ["${null_resource.tags_as_list_of_maps.*.triggers}"] + cluster_security_group_id = "${var.cluster_security_group_id == "" ? aws_security_group.cluster.id : var.cluster_security_group_id}" + worker_security_group_id = "${var.worker_security_group_id == "" ? aws_security_group.workers.id : var.worker_security_group_id}" + workstation_external_cidr = "${chomp(data.http.workstation_external_ip.body)}/32" + # Mapping from the node type that we selected and the max number of pods that it can run # Taken from https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-06-05/amazon-eks-nodegroup.yaml max_pod_per_node = { @@ -59,48 +64,123 @@ locals { x1.32xlarge = 234 } - asg_tags = ["${null_resource.tags_as_list_of_maps.*.triggers}"] - - # More information: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-06-05/amazon-eks-nodegroup.yaml - config_map_aws_auth = <