mirror of
https://github.com/ysoftdevs/terraform-aws-eks.git
synced 2026-03-29 21:51:51 +02:00
docs: Replace cluster autoscaler/node termination handler example with Karpenter (#1994)
This commit is contained in:
Bryant Biggs
248
examples/karpenter/main.tf
Normal file
248
examples/karpenter/main.tf
Normal file
@@ -0,0 +1,248 @@
|
||||
provider "aws" {
|
||||
region = local.region
|
||||
}
|
||||
|
||||
locals {
|
||||
name = "ex-${replace(basename(path.cwd), "_", "-")}"
|
||||
cluster_version = "1.22"
|
||||
region = "eu-west-1"
|
||||
|
||||
tags = {
|
||||
Example = local.name
|
||||
GithubRepo = "terraform-aws-eks"
|
||||
GithubOrg = "terraform-aws-modules"
|
||||
}
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# EKS Module
|
||||
################################################################################
|
||||
|
||||
module "eks" {
|
||||
source = "../.."
|
||||
|
||||
cluster_name = local.name
|
||||
cluster_version = local.cluster_version
|
||||
cluster_endpoint_private_access = true
|
||||
cluster_endpoint_public_access = true
|
||||
|
||||
vpc_id = module.vpc.vpc_id
|
||||
subnet_ids = module.vpc.private_subnets
|
||||
|
||||
node_security_group_additional_rules = {
|
||||
# Control plane invoke Karpenter webhook
|
||||
ingress_karpenter_webhook_tcp = {
|
||||
description = "Control plane invoke Karpenter webhook"
|
||||
protocol = "tcp"
|
||||
from_port = 8443
|
||||
to_port = 8443
|
||||
type = "ingress"
|
||||
source_cluster_security_group = true
|
||||
}
|
||||
}
|
||||
|
||||
eks_managed_node_groups = {
|
||||
karpenter = {
|
||||
instance_types = ["t3.medium"]
|
||||
|
||||
min_size = 1
|
||||
max_size = 2
|
||||
desired_size = 1
|
||||
|
||||
iam_role_additional_policies = [
|
||||
# Required by Karpenter
|
||||
"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
tags = merge(local.tags, {
|
||||
# NOTE - if creating multiple security groups with this module, only tag the
|
||||
# security group that Karpenter should utilize with the following tag
|
||||
# (i.e. - at most, only one security group should have this tag in your account)
|
||||
"karpenter.sh/discovery" = local.name
|
||||
})
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# Karpenter
|
||||
################################################################################
|
||||
|
||||
provider "helm" {
|
||||
kubernetes {
|
||||
host = module.eks.cluster_endpoint
|
||||
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
|
||||
|
||||
exec {
|
||||
api_version = "client.authentication.k8s.io/v1alpha1"
|
||||
command = "aws"
|
||||
# This requires the awscli to be installed locally where Terraform is executed
|
||||
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "kubectl" {
|
||||
apply_retry_count = 5
|
||||
host = module.eks.cluster_endpoint
|
||||
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
|
||||
load_config_file = false
|
||||
|
||||
exec {
|
||||
api_version = "client.authentication.k8s.io/v1alpha1"
|
||||
command = "aws"
|
||||
# This requires the awscli to be installed locally where Terraform is executed
|
||||
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
|
||||
}
|
||||
}
|
||||
|
||||
module "karpenter_irsa" {
|
||||
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
|
||||
version = "~> 4.15"
|
||||
|
||||
role_name = "karpenter-controller-${local.name}"
|
||||
attach_karpenter_controller_policy = true
|
||||
|
||||
karpenter_controller_cluster_id = module.eks.cluster_id
|
||||
karpenter_controller_node_iam_role_arns = [
|
||||
module.eks.eks_managed_node_groups["karpenter"].iam_role_arn
|
||||
]
|
||||
|
||||
oidc_providers = {
|
||||
ex = {
|
||||
provider_arn = module.eks.oidc_provider_arn
|
||||
namespace_service_accounts = ["karpenter:karpenter"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_instance_profile" "karpenter" {
|
||||
name = "KarpenterNodeInstanceProfile-${local.name}"
|
||||
role = module.eks.eks_managed_node_groups["karpenter"].iam_role_name
|
||||
}
|
||||
|
||||
resource "helm_release" "karpenter" {
|
||||
namespace = "karpenter"
|
||||
create_namespace = true
|
||||
|
||||
name = "karpenter"
|
||||
repository = "https://charts.karpenter.sh"
|
||||
chart = "karpenter"
|
||||
version = "0.8.1"
|
||||
|
||||
set {
|
||||
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
|
||||
value = module.karpenter_irsa.iam_role_arn
|
||||
}
|
||||
|
||||
set {
|
||||
name = "clusterName"
|
||||
value = module.eks.cluster_id
|
||||
}
|
||||
|
||||
set {
|
||||
name = "clusterEndpoint"
|
||||
value = module.eks.cluster_endpoint
|
||||
}
|
||||
|
||||
set {
|
||||
name = "aws.defaultInstanceProfile"
|
||||
value = aws_iam_instance_profile.karpenter.name
|
||||
}
|
||||
}
|
||||
|
||||
# Workaround - https://github.com/hashicorp/terraform-provider-kubernetes/issues/1380#issuecomment-967022975
|
||||
resource "kubectl_manifest" "karpenter_provisioner" {
|
||||
yaml_body = <<-YAML
|
||||
apiVersion: karpenter.sh/v1alpha5
|
||||
kind: Provisioner
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
requirements:
|
||||
- key: karpenter.sh/capacity-type
|
||||
operator: In
|
||||
values: ["spot"]
|
||||
limits:
|
||||
resources:
|
||||
cpu: 1000
|
||||
provider:
|
||||
subnetSelector:
|
||||
karpenter.sh/discovery: ${local.name}
|
||||
securityGroupSelector:
|
||||
karpenter.sh/discovery: ${local.name}
|
||||
tags:
|
||||
karpenter.sh/discovery: ${local.name}
|
||||
ttlSecondsAfterEmpty: 30
|
||||
YAML
|
||||
|
||||
depends_on = [
|
||||
helm_release.karpenter
|
||||
]
|
||||
}
|
||||
|
||||
# Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container)
|
||||
# and starts with zero replicas
|
||||
resource "kubectl_manifest" "karpenter_example_deployment" {
|
||||
yaml_body = <<-YAML
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: inflate
|
||||
spec:
|
||||
replicas: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app: inflate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: inflate
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 0
|
||||
containers:
|
||||
- name: inflate
|
||||
image: public.ecr.aws/eks-distro/kubernetes/pause:3.2
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1
|
||||
YAML
|
||||
|
||||
depends_on = [
|
||||
helm_release.karpenter
|
||||
]
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# Supporting Resources
|
||||
################################################################################
|
||||
|
||||
module "vpc" {
|
||||
source = "terraform-aws-modules/vpc/aws"
|
||||
version = "~> 3.0"
|
||||
|
||||
name = local.name
|
||||
cidr = "10.0.0.0/16"
|
||||
|
||||
azs = ["${local.region}a", "${local.region}b", "${local.region}c"]
|
||||
private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
|
||||
public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
|
||||
|
||||
enable_nat_gateway = true
|
||||
single_nat_gateway = true
|
||||
enable_dns_hostnames = true
|
||||
|
||||
public_subnet_tags = {
|
||||
"kubernetes.io/cluster/${local.name}" = "shared"
|
||||
"kubernetes.io/role/elb" = 1
|
||||
}
|
||||
|
||||
private_subnet_tags = {
|
||||
"kubernetes.io/cluster/${local.name}" = "shared"
|
||||
"kubernetes.io/role/internal-elb" = 1
|
||||
# Tags subnets for Karpenter auto-discovery
|
||||
"karpenter.sh/discovery" = local.name
|
||||
}
|
||||
|
||||
tags = local.tags
|
||||
}
|
||||
Reference in New Issue
Block a user