github-mirrors/terraform-aws-eks
1
0
Fork 0
mirror of https://github.com/ysoftdevs/terraform-aws-eks.git synced 2026-01-18 17:47:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

docs: Replace cluster autoscaler/node termination handler example with Karpenter (#1994)

Browse Source
This commit is contained in:
Bryant Biggs
2022-04-07 14:47:22 -04:00
committed by GitHub
parent 20af82846b
commit 3ff17205a4
13 changed files with 301 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.pre-commit-config.yaml
View File

@@ -23,7 +23,7 @@ repos:
- '--args=--only=terraform_standard_module_structure'
- '--args=--only=terraform_workspace_remote'
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.1.0
rev: v4.2.0
hooks:
- id: check-merge-conflict
- id: end-of-file-fixer

2
README.md
View File

@@ -162,7 +162,7 @@ module "eks" {
- [Complete](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/complete): EKS Cluster using all available node group types in various combinations demonstrating many of the supported features and configurations
- [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups
- [Fargate Profile](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/fargate_profile): EKS cluster using [Fargate Profiles](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html)
- [IRSA, Node Autoscaler, Instance Refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh): EKS Cluster using self-managed node group demonstrating how to enable/utilize instance refresh configuration along with node termination handler
- [Karpenter](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/karpenter): EKS Cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling
- [Self Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/self_managed_node_group): EKS Cluster using self-managed node groups
- [User Data](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/user_data): Various supported methods of providing necessary bootstrap scripts and configuration settings via user data

2
docs/faq.md
View File

@@ -52,8 +52,6 @@ TL;DR - Terraform resource passed into the modules map definition _must_ be know
- `node_security_group_additional_rules` (i.e. - referencing an external security group resource in a rule)
- `iam_role_additional_policies` (i.e. - referencing an external policy resource)
- Setting `instance_refresh_enabled = true` will recreate your worker nodes without draining them first. It is recommended to install [aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler) for proper node draining. See the [instance_refresh](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa_autoscale_refresh) example provided.
### Why are nodes not being registered?
Nodes not being able to register with the EKS control plane is generally due to networking mis-configurations.

2
examples/eks_managed_node_group/main.tf
View File

@@ -4,7 +4,7 @@ provider "aws" {
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.21"
cluster_version = "1.22"
region = "eu-west-1"
tags = {

2
examples/fargate_profile/main.tf
View File

@@ -4,7 +4,7 @@ provider "aws" {
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.21"
cluster_version = "1.22"
region = "eu-west-1"
tags = {

227
examples/irsa_autoscale_refresh/charts.tf
View File

@@ -1,227 +0,0 @@
provider "helm" {
kubernetes {
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
token = data.aws_eks_cluster_auth.cluster.token
}
}
################################################################################
# Cluster Autoscaler
# Based on the official docs at
# https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
################################################################################
resource "helm_release" "cluster_autoscaler" {
name = "cluster-autoscaler"
namespace = "kube-system"
repository = "https://kubernetes.github.io/autoscaler"
chart = "cluster-autoscaler"
version = "9.10.8"
create_namespace = false
set {
name = "awsRegion"
value = local.region
}
set {
name = "rbac.serviceAccount.name"
value = "cluster-autoscaler-aws"
}
set {
name = "rbac.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = module.cluster_autoscaler_irsa.iam_role_arn
type = "string"
}
set {
name = "autoDiscovery.clusterName"
value = local.name
}
set {
name = "autoDiscovery.enabled"
value = "true"
}
set {
name = "rbac.create"
value = "true"
}
depends_on = [
module.eks.cluster_id,
null_resource.apply,
]
}
module "cluster_autoscaler_irsa" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 4.12"
role_name_prefix = "cluster-autoscaler"
role_description = "IRSA role for cluster autoscaler"
attach_cluster_autoscaler_policy = true
cluster_autoscaler_cluster_ids = [module.eks.cluster_id]
oidc_providers = {
main = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:cluster-autoscaler-aws"]
}
}
tags = local.tags
}
################################################################################
# Node Termination Handler
# Based on the official docs at
# https://github.com/aws/aws-node-termination-handler
################################################################################
resource "helm_release" "aws_node_termination_handler" {
name = "aws-node-termination-handler"
namespace = "kube-system"
repository = "https://aws.github.io/eks-charts"
chart = "aws-node-termination-handler"
version = "0.16.0"
create_namespace = false
set {
name = "awsRegion"
value = local.region
}
set {
name = "serviceAccount.name"
value = "aws-node-termination-handler"
}
set {
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = module.node_termination_handler_irsa.iam_role_arn
type = "string"
}
set {
name = "enableSqsTerminationDraining"
value = "true"
}
set {
name = "enableSpotInterruptionDraining"
value = "true"
}
set {
name = "queueURL"
value = module.aws_node_termination_handler_sqs.sqs_queue_id
}
set {
name = "logLevel"
value = "debug"
}
depends_on = [
module.eks.cluster_id,
null_resource.apply,
]
}
module "node_termination_handler_irsa" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 4.12"
role_name_prefix = "node-termination-handler"
role_description = "IRSA role for node termination handler"
attach_node_termination_handler_policy = true
node_termination_handler_sqs_queue_arns = [module.aws_node_termination_handler_sqs.sqs_queue_arn]
oidc_providers = {
main = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:aws-node-termination-handler"]
}
}
tags = local.tags
}
module "aws_node_termination_handler_sqs" {
source = "terraform-aws-modules/sqs/aws"
version = "~> 3.0"
name = local.name
message_retention_seconds = 300
policy = data.aws_iam_policy_document.aws_node_termination_handler_sqs.json
tags = local.tags
}
data "aws_iam_policy_document" "aws_node_termination_handler_sqs" {
statement {
actions = ["sqs:SendMessage"]
resources = ["arn:aws:sqs:${local.region}:${data.aws_caller_identity.current.account_id}:${local.name}"]
principals {
type = "Service"
identifiers = [
"events.amazonaws.com",
"sqs.amazonaws.com",
]
}
}
}
resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_asg" {
name = "${local.name}-asg-termination"
description = "Node termination event rule"
event_pattern = jsonencode({
"source" : ["aws.autoscaling"],
"detail-type" : ["EC2 Instance-terminate Lifecycle Action"]
"resources" : [for group in module.eks.self_managed_node_groups : group.autoscaling_group_arn]
})
tags = local.tags
}
resource "aws_cloudwatch_event_target" "aws_node_termination_handler_asg" {
target_id = "${local.name}-asg-termination"
rule = aws_cloudwatch_event_rule.aws_node_termination_handler_asg.name
arn = module.aws_node_termination_handler_sqs.sqs_queue_arn
}
resource "aws_cloudwatch_event_rule" "aws_node_termination_handler_spot" {
name = "${local.name}-spot-termination"
description = "Node termination event rule"
event_pattern = jsonencode({
"source" : ["aws.ec2"],
"detail-type" : ["EC2 Spot Instance Interruption Warning"]
})
}
resource "aws_cloudwatch_event_target" "aws_node_termination_handler_spot" {
target_id = "${local.name}-spot-termination"
rule = aws_cloudwatch_event_rule.aws_node_termination_handler_spot.name
arn = module.aws_node_termination_handler_sqs.sqs_queue_arn
}
# Creating the lifecycle-hook outside of the ASG resource's `initial_lifecycle_hook`
# ensures that node termination does not require the lifecycle action to be completed,
# and thus allows the ASG to be destroyed cleanly.
resource "aws_autoscaling_lifecycle_hook" "aws_node_termination_handler" {
for_each = module.eks.self_managed_node_groups
name = "aws-node-termination-handler-${each.value.autoscaling_group_name}"
autoscaling_group_name = each.value.autoscaling_group_name
lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING"
heartbeat_timeout = 300
default_result = "CONTINUE"
}

188
examples/irsa_autoscale_refresh/main.tf
View File

@@ -1,188 +0,0 @@
provider "aws" {
region = local.region
}
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.21"
region = "eu-west-1"
tags = {
Example = local.name
GithubRepo = "terraform-aws-eks"
GithubOrg = "terraform-aws-modules"
}
}
data "aws_caller_identity" "current" {}
data "aws_eks_cluster_auth" "cluster" {
name = module.eks.cluster_id
}
################################################################################
# EKS Module
################################################################################
module "eks" {
source = "../.."
cluster_name = local.name
cluster_version = local.cluster_version
cluster_endpoint_private_access = true
cluster_endpoint_public_access = true
vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.private_subnets
# Self Managed Node Group(s)
self_managed_node_groups = {
refresh = {
max_size = 5
desired_size = 1
instance_type = "m5.large"
instance_refresh = {
strategy = "Rolling"
preferences = {
checkpoint_delay = 600
checkpoint_percentages = [35, 70, 100]
instance_warmup = 300
min_healthy_percentage = 50
}
triggers = ["tag"]
}
tags = { "aws-node-termination-handler/managed" = "true" }
}
mixed_instance = {
use_mixed_instances_policy = true
mixed_instances_policy = {
instances_distribution = {
on_demand_base_capacity = 0
on_demand_percentage_above_base_capacity = 10
spot_allocation_strategy = "capacity-optimized"
}
override = [
{
instance_type = "m5.large"
weighted_capacity = "1"
},
{
instance_type = "m6i.large"
weighted_capacity = "2"
},
]
}
tags = { "aws-node-termination-handler/managed" = "true" }
}
spot = {
instance_type = "m5.large"
instance_market_options = {
market_type = "spot"
}
bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'"
tags = { "aws-node-termination-handler/managed" = "true" }
}
}
tags = merge(local.tags, { Foo = "bar" })
}
################################################################################
# aws-auth configmap
# Only EKS managed node groups automatically add roles to aws-auth configmap
# so we need to ensure fargate profiles and self-managed node roles are added
################################################################################
data "aws_eks_cluster_auth" "this" {
name = module.eks.cluster_id
}
locals {
kubeconfig = yamlencode({
apiVersion = "v1"
kind = "Config"
current-context = "terraform"
clusters = [{
name = module.eks.cluster_id
cluster = {
certificate-authority-data = module.eks.cluster_certificate_authority_data
server = module.eks.cluster_endpoint
}
}]
contexts = [{
name = "terraform"
context = {
cluster = module.eks.cluster_id
user = "terraform"
}
}]
users = [{
name = "terraform"
user = {
token = data.aws_eks_cluster_auth.this.token
}
}]
})
}
resource "null_resource" "apply" {
triggers = {
kubeconfig = base64encode(local.kubeconfig)
cmd_patch = <<-EOT
kubectl create configmap aws-auth -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode)
kubectl patch configmap/aws-auth --patch "${module.eks.aws_auth_configmap_yaml}" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode)
EOT
}
provisioner "local-exec" {
interpreter = ["/bin/bash", "-c"]
environment = {
KUBECONFIG = self.triggers.kubeconfig
}
command = self.triggers.cmd_patch
}
}
################################################################################
# Supporting Resources
################################################################################
module "vpc" {
source = "terraform-aws-modules/vpc/aws"
version = "~> 3.0"
name = local.name
cidr = "10.0.0.0/16"
azs = ["${local.region}a", "${local.region}b", "${local.region}c"]
private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
enable_nat_gateway = true
single_nat_gateway = true
enable_dns_hostnames = true
enable_flow_log = true
create_flow_log_cloudwatch_iam_role = true
create_flow_log_cloudwatch_log_group = true
public_subnet_tags = {
"kubernetes.io/cluster/${local.name}" = "shared"
"kubernetes.io/role/elb" = 1
}
private_subnet_tags = {
"kubernetes.io/cluster/${local.name}" = "shared"
"kubernetes.io/role/internal-elb" = 1
}
tags = local.tags
}

68
examples/irsa_autoscale_refresh/README.md → examples/karpenter/README.md
View File

@@ -1,10 +1,6 @@
# IRSA, Cluster Autoscaler, and Instance Refresh example
# Karpenter Example
Configuration in this directory creates an AWS EKS cluster with:
- [IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) enabled
- [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) provisioned via a Helm Chart manifest
- [Instance Refresh](https://aws.amazon.com/blogs/compute/introducing-instance-refresh-for-ec2-auto-scaling/) feature for self managed node groups
- [Node Termination Handler](https://github.com/aws/aws-node-termination-handler) provisioned via a Helm Chart manifest
Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling.
## Usage
@@ -16,6 +12,38 @@ $ terraform plan
$ terraform apply
```
Once the cluster is up and running, you can check that Karpenter is functioning as intended with the following command:
```bash
# First, make sure you have updated your local kubeconfig
aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter
# Second, scale the example deployment
kubectl scale deployment inflate --replicas 5
# You can watch Karpenter's controller logs with
kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller
```
You should see a new node named `karpenter.sh/provisioner-name/default` eventually come up in the console; this was provisioned by Karpenter in response to the scaled deployment above.
### Tear Down & Clean-Up
Because Karpenter manages the state of node resources outside of Terraform, Karpenter created resources will need to be de-provisioned first before removing the remaining resources with Terraform.
1. Remove the example deployment created above and any nodes created by Karpenter
```bash
kubectl delete deployment inflate
kubectl delete node -l karpenter.sh/provisioner-name=default
```
2. Remove the resources created by Terraform
```bash
terraform destroy
```
Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources.
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
@@ -25,43 +53,33 @@ Note that this example may create resources which cost money. Run `terraform des
|------|---------|
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.0 |
| <a name="requirement_null"></a> [null](#requirement\_null) | >= 3.0 |
| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.4 |
| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14 |
## Providers
| Name | Version |
|------|---------|
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
| <a name="provider_helm"></a> [helm](#provider\_helm) | >= 2.0 |
| <a name="provider_null"></a> [null](#provider\_null) | >= 3.0 |
| <a name="provider_helm"></a> [helm](#provider\_helm) | >= 2.4 |
| <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | >= 1.14 |
## Modules
| Name | Source | Version |
|------|--------|---------|
| <a name="module_aws_node_termination_handler_sqs"></a> [aws\_node\_termination\_handler\_sqs](#module\_aws\_node\_termination\_handler\_sqs) | terraform-aws-modules/sqs/aws | ~> 3.0 |
| <a name="module_cluster_autoscaler_irsa"></a> [cluster\_autoscaler\_irsa](#module\_cluster\_autoscaler\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 |
| <a name="module_eks"></a> [eks](#module\_eks) | ../.. | n/a |
| <a name="module_node_termination_handler_irsa"></a> [node\_termination\_handler\_irsa](#module\_node\_termination\_handler\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.12 |
| <a name="module_karpenter_irsa"></a> [karpenter\_irsa](#module\_karpenter\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.15 |
| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 |
## Resources
| Name | Type |
|------|------|
| [aws_autoscaling_lifecycle_hook.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_lifecycle_hook) | resource |
| [aws_cloudwatch_event_rule.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_rule.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_target.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_cloudwatch_event_target.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
| [aws_iam_policy_document.aws_node_termination_handler_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_instance_profile.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
| [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
## Inputs

248
examples/karpenter/main.tf Normal file
View File

@@ -0,0 +1,248 @@
provider "aws" {
region = local.region
}
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.22"
region = "eu-west-1"
tags = {
Example = local.name
GithubRepo = "terraform-aws-eks"
GithubOrg = "terraform-aws-modules"
}
}
################################################################################
# EKS Module
################################################################################
module "eks" {
source = "../.."
cluster_name = local.name
cluster_version = local.cluster_version
cluster_endpoint_private_access = true
cluster_endpoint_public_access = true
vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.private_subnets
node_security_group_additional_rules = {
# Control plane invoke Karpenter webhook
ingress_karpenter_webhook_tcp = {
description = "Control plane invoke Karpenter webhook"
protocol = "tcp"
from_port = 8443
to_port = 8443
type = "ingress"
source_cluster_security_group = true
}
}
eks_managed_node_groups = {
karpenter = {
instance_types = ["t3.medium"]
min_size = 1
max_size = 2
desired_size = 1
iam_role_additional_policies = [
# Required by Karpenter
"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
]
}
}
tags = merge(local.tags, {
# NOTE - if creating multiple security groups with this module, only tag the
# security group that Karpenter should utilize with the following tag
# (i.e. - at most, only one security group should have this tag in your account)
"karpenter.sh/discovery" = local.name
})
}
################################################################################
# Karpenter
################################################################################
provider "helm" {
kubernetes {
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
exec {
api_version = "client.authentication.k8s.io/v1alpha1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
}
}
}
provider "kubectl" {
apply_retry_count = 5
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
load_config_file = false
exec {
api_version = "client.authentication.k8s.io/v1alpha1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
}
}
module "karpenter_irsa" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 4.15"
role_name = "karpenter-controller-${local.name}"
attach_karpenter_controller_policy = true
karpenter_controller_cluster_id = module.eks.cluster_id
karpenter_controller_node_iam_role_arns = [
module.eks.eks_managed_node_groups["karpenter"].iam_role_arn
]
oidc_providers = {
ex = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["karpenter:karpenter"]
}
}
}
resource "aws_iam_instance_profile" "karpenter" {
name = "KarpenterNodeInstanceProfile-${local.name}"
role = module.eks.eks_managed_node_groups["karpenter"].iam_role_name
}
resource "helm_release" "karpenter" {
namespace = "karpenter"
create_namespace = true
name = "karpenter"
repository = "https://charts.karpenter.sh"
chart = "karpenter"
version = "0.8.1"
set {
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = module.karpenter_irsa.iam_role_arn
}
set {
name = "clusterName"
value = module.eks.cluster_id
}
set {
name = "clusterEndpoint"
value = module.eks.cluster_endpoint
}
set {
name = "aws.defaultInstanceProfile"
value = aws_iam_instance_profile.karpenter.name
}
}
# Workaround - https://github.com/hashicorp/terraform-provider-kubernetes/issues/1380#issuecomment-967022975
resource "kubectl_manifest" "karpenter_provisioner" {
yaml_body = <<-YAML
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
name: default
spec:
requirements:
- key: karpenter.sh/capacity-type
operator: In
values: ["spot"]
limits:
resources:
cpu: 1000
provider:
subnetSelector:
karpenter.sh/discovery: ${local.name}
securityGroupSelector:
karpenter.sh/discovery: ${local.name}
tags:
karpenter.sh/discovery: ${local.name}
ttlSecondsAfterEmpty: 30
YAML
depends_on = [
helm_release.karpenter
]
}
# Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container)
# and starts with zero replicas
resource "kubectl_manifest" "karpenter_example_deployment" {
yaml_body = <<-YAML
apiVersion: apps/v1
kind: Deployment
metadata:
name: inflate
spec:
replicas: 0
selector:
matchLabels:
app: inflate
template:
metadata:
labels:
app: inflate
spec:
terminationGracePeriodSeconds: 0
containers:
- name: inflate
image: public.ecr.aws/eks-distro/kubernetes/pause:3.2
resources:
requests:
cpu: 1
YAML
depends_on = [
helm_release.karpenter
]
}
################################################################################
# Supporting Resources
################################################################################
module "vpc" {
source = "terraform-aws-modules/vpc/aws"
version = "~> 3.0"
name = local.name
cidr = "10.0.0.0/16"
azs = ["${local.region}a", "${local.region}b", "${local.region}c"]
private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
enable_nat_gateway = true
single_nat_gateway = true
enable_dns_hostnames = true
public_subnet_tags = {
"kubernetes.io/cluster/${local.name}" = "shared"
"kubernetes.io/role/elb" = 1
}
private_subnet_tags = {
"kubernetes.io/cluster/${local.name}" = "shared"
"kubernetes.io/role/internal-elb" = 1
# Tags subnets for Karpenter auto-discovery
"karpenter.sh/discovery" = local.name
}
tags = local.tags
}

0
examples/irsa_autoscale_refresh/outputs.tf → examples/karpenter/outputs.tf
View File

0
examples/irsa_autoscale_refresh/variables.tf → examples/karpenter/variables.tf
View File

10
examples/irsa_autoscale_refresh/versions.tf → examples/karpenter/versions.tf
View File

@@ -6,13 +6,13 @@ terraform {
source = "hashicorp/aws"
version = ">= 3.72"
}
null = {
source = "hashicorp/null"
version = ">= 3.0"
}
helm = {
source = "hashicorp/helm"
version = ">= 2.0"
version = ">= 2.4"
}
kubectl = {
source = "gavinbunney/kubectl"
version = ">= 1.14"
}
}
}

2
examples/self_managed_node_group/main.tf
View File

@@ -4,7 +4,7 @@ provider "aws" {
locals {
name = "ex-${replace(basename(path.cwd), "_", "-")}"
cluster_version = "1.21"
cluster_version = "1.22"
region = "eu-west-1"
tags = {