github-mirrors/terraform-aws-eks
1
0
Fork 0
mirror of https://github.com/ysoftdevs/terraform-aws-eks.git synced 2026-05-01 04:44:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat!: Add support for Outposts, remove node security group, add support for addon preserve and most_recent configurations (#2250)

Browse Source 
Co-authored-by: Anton Babenko <anton@antonbabenko.com>
Resolves undefined
This commit is contained in:
Bryant Biggs
2022-12-05 16:26:23 -05:00
committed by GitHub
parent efbe952632
commit b2e97ca3dc
66 changed files with 2749 additions and 1776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

307
examples/self_managed_node_group/main.tf
View File

@@ -14,11 +14,17 @@ provider "kubernetes" {
}
}
data "aws_caller_identity" "current" {}
data "aws_availability_zones" "available" {}
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.24"
region = "eu-west-1"
vpc_cidr = "10.0.0.0/16"
azs = slice(data.aws_availability_zones.available.names, 0, 3)
tags = {
Example = local.name
GithubRepo = "terraform-aws-eks"
@@ -26,8 +32,6 @@ locals {
}
}
data "aws_caller_identity" "current" {}
################################################################################
# EKS Module
################################################################################
@@ -35,69 +39,31 @@ data "aws_caller_identity" "current" {}
module "eks" {
source = "../.."
cluster_name = local.name
cluster_version = local.cluster_version
cluster_endpoint_private_access = true
cluster_endpoint_public_access = true
cluster_name = local.name
cluster_version = local.cluster_version
cluster_endpoint_public_access = true
cluster_addons = {
coredns = {
resolve_conflicts = "OVERWRITE"
most_recent = true
}
kube-proxy = {
most_recent = true
}
kube-proxy = {}
vpc-cni = {
resolve_conflicts = "OVERWRITE"
most_recent = true
}
}
cluster_encryption_config = [{
provider_key_arn = aws_kms_key.eks.arn
resources = ["secrets"]
}]
vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.private_subnets
vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.private_subnets
control_plane_subnet_ids = module.vpc.intra_subnets
# Self managed node groups will not automatically create the aws-auth configmap so we need to
create_aws_auth_configmap = true
manage_aws_auth_configmap = true
# Extend cluster security group rules
cluster_security_group_additional_rules = {
egress_nodes_ephemeral_ports_tcp = {
description = "To node 1025-65535"
protocol = "tcp"
from_port = 1025
to_port = 65535
type = "egress"
source_node_security_group = true
}
}
# Extend node-to-node security group rules
node_security_group_additional_rules = {
ingress_self_all = {
description = "Node to node all ports/protocols"
protocol = "-1"
from_port = 0
to_port = 0
type = "ingress"
self = true
}
egress_all = {
description = "Node all egress"
protocol = "-1"
from_port = 0
to_port = 0
type = "egress"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
}
self_managed_node_group_defaults = {
create_security_group = false
# enable discovery of autoscaling groups by cluster-autoscaler
autoscaling_group_tags = {
"k8s.io/cluster-autoscaler/enabled" : true,
@@ -117,24 +83,31 @@ module "eks" {
ami_id = data.aws_ami.eks_default_bottlerocket.id
instance_type = "m5.large"
desired_size = 2
key_name = aws_key_pair.this.key_name
iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"]
key_name = module.key_pair.key_pair_name
bootstrap_extra_args = <<-EOT
# The admin host container provides SSH access and runs with "superpowers".
# It is disabled by default, but can be disabled explicitly.
[settings.host-containers.admin]
enabled = false
# The admin host container provides SSH access and runs with "superpowers".
# It is disabled by default, but can be disabled explicitly.
[settings.host-containers.admin]
enabled = false
# The control host container provides out-of-band access via SSM.
# It is enabled by default, and can be disabled if you do not expect to use SSM.
# This could leave you with no way to access the API and change settings on an existing node!
[settings.host-containers.control]
enabled = true
# The control host container provides out-of-band access via SSM.
# It is enabled by default, and can be disabled if you do not expect to use SSM.
# This could leave you with no way to access the API and change settings on an existing node!
[settings.host-containers.control]
enabled = true
[settings.kubernetes.node-labels]
ingress = "allowed"
# extra args added
[settings.kernel]
lockdown = "integrity"
[settings.kubernetes.node-labels]
label1 = "foo"
label2 = "bar"
[settings.kubernetes.node-taints]
dedicated = "experimental:PreferNoSchedule"
special = "true:NoSchedule"
EOT
}
@@ -177,15 +150,14 @@ module "eks" {
instance_type = "c5n.9xlarge"
post_bootstrap_user_data = <<-EOT
# Install EFA
curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz
tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer
./efa_installer.sh -y --minimal
fi_info -p efa -t FI_EP_RDM
# Install EFA
curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz
tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer
./efa_installer.sh -y --minimal
fi_info -p efa -t FI_EP_RDM
# Disable ptrace
sysctl -w kernel.yama.ptrace_scope=0
# Disable ptrace
sysctl -w kernel.yama.ptrace_scope=0
EOT
network_interfaces = [
@@ -214,12 +186,12 @@ module "eks" {
bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'"
pre_bootstrap_user_data = <<-EOT
export CONTAINER_RUNTIME="containerd"
export USE_MAX_PODS=false
export CONTAINER_RUNTIME="containerd"
export USE_MAX_PODS=false
EOT
post_bootstrap_user_data = <<-EOT
echo "you are free little kubelet!"
echo "you are free little kubelet!"
EOT
instance_type = "m6i.large"
@@ -228,9 +200,8 @@ module "eks" {
launch_template_use_name_prefix = true
launch_template_description = "Self managed node group example launch template"
ebs_optimized = true
vpc_security_group_ids = [aws_security_group.additional.id]
enable_monitoring = true
ebs_optimized = true
enable_monitoring = true
block_device_mappings = {
xvda = {
@@ -241,7 +212,7 @@ module "eks" {
iops = 3000
throughput = 150
encrypted = true
kms_key_id = aws_kms_key.ebs.arn
kms_key_id = module.ebs_kms_key.key_id
delete_on_termination = true
}
}
@@ -267,34 +238,9 @@ module "eks" {
iam_role_tags = {
Purpose = "Protector of the kubelet"
}
iam_role_additional_policies = [
"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
]
create_security_group = true
security_group_name = "self-managed-node-group-complete-example"
security_group_use_name_prefix = false
security_group_description = "Self managed node group complete example security group"
security_group_rules = {
phoneOut = {
description = "Hello CloudFlare"
protocol = "udp"
from_port = 53
to_port = 53
type = "egress"
cidr_blocks = ["1.1.1.1/32"]
}
phoneHome = {
description = "Hello cluster"
protocol = "udp"
from_port = 53
to_port = 53
type = "egress"
source_cluster_security_group = true # bit of reflection lookup
}
}
security_group_tags = {
Purpose = "Protector of the kubelet"
iam_role_additional_policies = {
AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
additional = aws_iam_policy.additional.arn
}
timeouts = {
@@ -321,11 +267,12 @@ module "vpc" {
version = "~> 3.0"
name = local.name
cidr = "10.0.0.0/16"
cidr = local.vpc_cidr
azs = ["${local.region}a", "${local.region}b", "${local.region}c"]
private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
azs = local.azs
private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]
public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)]
intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)]
enable_nat_gateway = true
single_nat_gateway = true
@@ -336,44 +283,16 @@ module "vpc" {
create_flow_log_cloudwatch_log_group = true
public_subnet_tags = {
"kubernetes.io/cluster/${local.name}" = "shared"
"kubernetes.io/role/elb" = 1
"kubernetes.io/role/elb" = 1
}
private_subnet_tags = {
"kubernetes.io/cluster/${local.name}" = "shared"
"kubernetes.io/role/internal-elb" = 1
"kubernetes.io/role/internal-elb" = 1
}
tags = local.tags
}
resource "aws_security_group" "additional" {
name_prefix = "${local.name}-additional"
vpc_id = module.vpc.vpc_id
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = [
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
]
}
tags = local.tags
}
resource "aws_kms_key" "eks" {
description = "EKS Secret Encryption Key"
deletion_window_in_days = 7
enable_key_rotation = true
tags = local.tags
}
data "aws_ami" "eks_default" {
most_recent = true
owners = ["amazon"]
@@ -394,19 +313,37 @@ data "aws_ami" "eks_default_bottlerocket" {
}
}
resource "tls_private_key" "this" {
algorithm = "RSA"
module "key_pair" {
source = "terraform-aws-modules/key-pair/aws"
version = "~> 2.0"
key_name_prefix = local.name
create_private_key = true
tags = local.tags
}
resource "aws_key_pair" "this" {
key_name = local.name
public_key = tls_private_key.this.public_key_openssh
}
module "ebs_kms_key" {
source = "terraform-aws-modules/kms/aws"
version = "~> 1.1"
resource "aws_kms_key" "ebs" {
description = "Customer managed key to encrypt self managed node group volumes"
deletion_window_in_days = 7
policy = data.aws_iam_policy_document.ebs.json
description = "Customer managed key to encrypt EKS managed node group volumes"
# Policy
key_administrators = [
data.aws_caller_identity.current.arn
]
key_service_users = [
# required for the ASG to manage encrypted volumes for nodes
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling",
# required for the cluster / persistentvolume-controller to create encrypted PVCs
module.eks.cluster_iam_role_arn,
]
# Aliases
aliases = ["eks/${local.name}/ebs"]
tags = local.tags
}
resource "aws_ec2_capacity_reservation" "targeted" {
@@ -417,58 +354,22 @@ resource "aws_ec2_capacity_reservation" "targeted" {
instance_match_criteria = "targeted"
}
# This policy is required for the KMS key used for EKS root volumes, so the cluster is allowed to enc/dec/attach encrypted EBS volumes
data "aws_iam_policy_document" "ebs" {
# Copy of default KMS policy that lets you manage it
statement {
sid = "Enable IAM User Permissions"
actions = ["kms:*"]
resources = ["*"]
resource "aws_iam_policy" "additional" {
name = "${local.name}-additional"
description = "Example usage of node additional policy"
principals {
type = "AWS"
identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
}
}
# Required for EKS
statement {
sid = "Allow service-linked role use of the CMK"
actions = [
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = [
"ec2:Describe*",
]
Effect = "Allow"
Resource = "*"
},
]
resources = ["*"]
})
principals {
type = "AWS"
identifiers = [
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes
module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs
]
}
}
statement {
sid = "Allow attachment of persistent resources"
actions = ["kms:CreateGrant"]
resources = ["*"]
principals {
type = "AWS"
identifiers = [
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", # required for the ASG to manage encrypted volumes for nodes
module.eks.cluster_iam_role_arn, # required for the cluster / persistentvolume-controller to create encrypted PVCs
]
}
condition {
test = "Bool"
variable = "kms:GrantIsForAWSResource"
values = ["true"]
}
}
tags = local.tags
}