starred/nix-config-ryan4yin
1
0
Fork 0
mirror of https://github.com/ryan4yin/nix-config.git synced 2026-05-30 18:20:38 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat: kubevirt on k3s

Browse Source
This commit is contained in:
Ryan Yin
2024-02-24 23:26:30 +08:00
parent 7d56db3e47
commit 9914644189
53 changed files with 8246 additions and 1382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pulumi/k3s-prod-1/.gitignore
+2
View File
@@ -0,0 +1,2 @@
*.pyc
venv/
pulumi/k3s-prod-1/Pulumi.yaml
+5 -2
View File
@@ -1,3 +1,6 @@
name: k3s-prod-1
runtime: go
description: A Go program to deploy all the resources for the k3s-prod-1 cluster
runtime:
name: python
options:
virtualenv: venv
description: A Python program to deploy a Helm chart onto a Kubernetes cluster
pulumi/k3s-prod-1/__main__.py
+3
View File
@@ -0,0 +1,3 @@
from monitoring import *
from networking import *
from visualization import *
pulumi/k3s-prod-1/go.mod
-103
View File
@@ -1,103 +0,0 @@
module k3s-prod-1
go 1.21
toolchain go1.21.6
require (
github.com/pulumi/pulumi-kubernetes v1.6.0
github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.7.1
github.com/pulumi/pulumi/sdk/v3 v3.106.0
)
require (
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/charmbracelet/bubbles v0.16.1 // indirect
github.com/charmbracelet/bubbletea v0.24.2 // indirect
github.com/charmbracelet/lipgloss v0.7.1 // indirect
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/djherbis/times v1.5.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-git/go-git/v5 v5.11.0 // indirect
github.com/gofrs/flock v0.7.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/hcl/v2 v2.17.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.2 // indirect
github.com/opentracing/basictracer-go v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pgavlin/fx v0.1.6 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.1.0 // indirect
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
github.com/pulumi/esc v0.6.2 // indirect
github.com/pulumi/pulumi/sdk v0.0.0-20200324171821-8ce10e1dfe54 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/skeema/knownhosts v1.2.1 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/src-d/gcfg v1.4.0 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/zclconf/go-cty v1.13.2 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.15.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
google.golang.org/grpc v1.57.1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/klog v1.0.0 // indirect
lukechampine.com/frand v1.4.2 // indirect
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 // indirect
)
pulumi/k3s-prod-1/go.sum
-1009
View File
File diff suppressed because it is too large Load Diff
pulumi/k3s-prod-1/main.go
-21
View File
@@ -1,21 +0,0 @@
package main
import (
"k3s-prod-1/monitoring"
"k3s-prod-1/networking"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
if err := monitoring.NewMonitoring(ctx, "prod"); err != nil {
return err
}
if err := networking.NewNetworking(ctx, "prod"); err != nil {
return err
}
return nil
})
}
pulumi/k3s-prod-1/monitoring/__init__.py
+1
View File
@@ -0,0 +1 @@
from .victoria_metrics import *
pulumi/k3s-prod-1/monitoring/monitoring.go
-28
View File
@@ -1,28 +0,0 @@
package monitoring
import (
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func NewMonitoring(ctx *pulumi.Context, env string) error {
// Create a Kubernetes Namespace
namespaceName := "monitoring"
namespace, err := corev1.NewNamespace(ctx, namespaceName, &corev1.NamespaceArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String(namespaceName),
},
})
if err != nil {
return err
}
// Export the name of the namespace
ctx.Export("monitoringNamespaceName", namespace.Metadata.Name())
if err := NewVictoriaMetrics(ctx, env, namespace); err != nil {
return err
}
return nil
}
pulumi/k3s-prod-1/monitoring/victoria_metrics.py
+42
View File
@@ -0,0 +1,42 @@
import pulumi
import pulumi_kubernetes as kubernetes
from pathlib import Path
import yaml
config = pulumi.Config()
k8s_namespace = "monitoring"
app_labels = {
"app": "monitoring",
}
victoriaMetricsvaluesPath = Path(__file__).parent / "victoria_metrics_helm_values.yml"
# Create a namespace (user supplies the name of the namespace)
monitoring_ns = kubernetes.core.v1.Namespace(
"monitoring",
metadata=kubernetes.meta.v1.ObjectMetaArgs(
labels=app_labels,
name=k8s_namespace,
),
)
# https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack
victoriaMetrics = kubernetes.helm.v3.Release(
"victoria-metrics-k8s-stack",
chart="victoria-metrics-k8s-stack",
namespace=monitoring_ns.metadata.name,
repository_opts=kubernetes.helm.v3.RepositoryOptsArgs(
repo="https://victoriametrics.github.io/helm-charts/",
),
version="0.19.2",
skip_crds=False,
atomic=True, # purges chart on fail
cleanup_on_fail=True, # Allow deletion of new resources created in this upgrade when upgrade fails.
dependency_update=True, # run helm dependency update before installing the chart
reset_values=True, # When upgrading, reset the values to the ones built into the chart
# verify=True, # verify the package before installing it
# recreate_pods=True, # performs pods restart for the resource if applicable
value_yaml_files=[pulumi.FileAsset(victoriaMetricsvaluesPath)],
)
pulumi/k3s-prod-1/monitoring/victoria_metrics_helm_values.yml
+37
View File
@@ -0,0 +1,37 @@
# https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack
#
# Pulumi will complain ` ValueError: unexpected input of type set` if some values are not available in helm chart!
grafana:
enabled: true
defaultDashboardsTimezone: utc+8
ingress:
enabled: true
hosts:
- k8s-grafana.writefor.fun
persistence:
type: pvc
enabled: false
kube-state-metrics:
enabled: true
prometheus-node-exporter:
# install node exporter via nixos, not container
enabled: false
vmagent:
# vmagent collects metrics from targets and sends them to a remote storage
enabled: true
vmalert:
# vmalert is a Prometheus-compatible alertmanager
enabled: true
vmsingle:
# Single-node VictoriaMetrics for storing metrics.
# https://docs.victoriametrics.com/faq/#which-victoriametrics-type-is-recommended-for-use-in-production---single-node-or-cluster
# vmsingle = vmcluster(vmselect + vmstorage + vminsert)
enabled: true
ingress:
hosts:
- vm.writefor.fun
spec:
storage:
resources:
requests:
storage: 50Gi
pulumi/k3s-prod-1/monitoring/victoriametrics.go
-85
View File
@@ -1,85 +0,0 @@
package monitoring
import (
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func NewVictoriaMetrics(ctx *pulumi.Context, env string, namespace corev1.Namespace) error {
var opts []pulumi.ResourceOption
opts = append(opts, pulumi.DependsOn([]pulumi.Resource{namespace}))
// https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack
_, err := helm.NewChart(ctx, "victoria-metrics-k8s-stack", helm.ChartArgs{
Chart: pulumi.String("victoria-metrics-k8s-stack"),
Version: pulumi.String("0.19.0"),
Namespace: pulumi.String(namespace.Metadata.Name()),
FetchArgs: helm.FetchArgs{
Repo: pulumi.String("https://victoriametrics.github.io/helm-charts/"),
},
// https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-k8s-stack/README.md
Values: pulumi.Map{
// grafana.ingress.enabled: true
"ingress": pulumi.Map{
"enabled": pulumi.Bool(true),
},
// grafana.defaultDashboardsTimezone: utc+8
// grafana.ingress.hosts[0].host: grafana.example.com
"grafana": pulumi.Map{
"defaultDashboardsTimezone": pulumi.String("utc+8"),
"ingress": pulumi.Map{
"hosts": pulumi.Array{
pulumi.Map{
"host": pulumi.String("k8s-grafana.writefor.fun"),
},
},
},
},
// prometheus-node-exporter.enabled: false
"nodeExporter": pulumi.Map{
"enabled": pulumi.Bool(false),
},
"vmsingle": pulumi.Map{
"enabled": pulumi.Bool(true),
"ingress": pulumi.Map{
"hosts": pulumi.Array{
pulumi.Map{
"host": pulumi.String("vm.writefor.fun"),
},
},
},
// https://docs.victoriametrics.com/operator/api/#vmsinglespec
"spec": pulumi.Map{
"affinity": pulumi.Map{
"nodeAffinity": pulumi.Map{
"requiredDuringSchedulingIgnoredDuringExecution": pulumi.Map{
"nodeSelectorTerms": pulumi.Array{
pulumi.Map{
"matchExpressions": pulumi.Array{
pulumi.Map{
"key": pulumi.String("kubernetes.io/arch"),
"operator": pulumi.String("In"),
"values": pulumi.Array{
pulumi.String("amd64"),
},
},
},
},
},
},
},
},
"storage": pulumi.Map{
"resources": pulumi.Map{
"requests": pulumi.Map{
"storage": pulumi.String("50Gi"),
},
},
},
},
},
},
}, opts...)
return err
}
pulumi/k3s-prod-1/networking/__init__.py
+1
View File
@@ -0,0 +1 @@
from .cert_manager import *
pulumi/k3s-prod-1/networking/cert-manager.go
-25
View File
@@ -1,25 +0,0 @@
package networking
import (
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func NewCertManager(ctx *pulumi.Context, env string, namespace corev1.Namespace) error {
var opts []pulumi.ResourceOption
opts = append(opts, pulumi.DependsOn([]pulumi.Resource{namespace}))
_, err := helm.NewChart(ctx, "cert-manager", helm.ChartArgs{
Chart: pulumi.String("cert-manager"),
Version: pulumi.String("corev1.14.2 "),
Namespace: pulumi.String(namespace.Metadata.Name()),
FetchArgs: helm.FetchArgs{
Repo: pulumi.String("https://charts.jetstack.io"),
},
// https://cert-manager.io/docs/installation/helm/
Values: pulumi.Map{},
}, opts...)
return err
}
pulumi/k3s-prod-1/networking/cert_manager.py
+15
View File
@@ -0,0 +1,15 @@
import pulumi
from pulumi_kubernetes.core.v1 import Namespace
from pulumi_kubernetes_cert_manager import CertManager, ReleaseArgs
ns_name = "cert-manager"
ns = Namespace("cert-manager", metadata={"name": ns_name})
# Install cert-manager into our cluster.
manager = CertManager(
"cert-manager",
install_crds=True,
helm_options=ReleaseArgs(
namespace=ns_name,
),
)
pulumi/k3s-prod-1/networking/networking.go
-28
View File
@@ -1,28 +0,0 @@
package networking
import (
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func NewNetworking(ctx *pulumi.Context, env string) error {
// Create a Kubernetes Namespace
namespaceName := "networking"
namespace, err := corev1.NewNamespace(ctx, namespaceName, &corev1.NamespaceArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String(namespaceName),
},
})
if err != nil {
return err
}
// Export the name of the namespace
ctx.Export("networkingNamespaceName", namespace.Metadata.Name())
if err := NewCertManager(ctx, env, namespace); err != nil {
return err
}
return nil
}
pulumi/k3s-prod-1/requirements.txt
+2
View File
@@ -0,0 +1,2 @@
pulumi>=3.0.0,<4.0.0
pulumi-kubernetes>=4.0.0,<5.0.0
pulumi/k3s-prod-1/visualization/__init__.py
+2
View File
@@ -0,0 +1,2 @@
from .kubevirt import *
from .virtual_machines import *
pulumi/k3s-prod-1/visualization/kubevirt.py
+22
View File
@@ -0,0 +1,22 @@
from pulumi_kubernetes.yaml import ConfigGroup
from pathlib import Path
currentDir = Path(__file__).parent
def virtHandlerNodePlacement(obj, opts):
if obj["kind"] == "KubeVirt":
obj["spec"]["workloads"] = {
"nodePlacement": {
"nodeSelector": {"node-type": "worker"}
}
}
kubevirt = ConfigGroup(
"kubevirt",
files=[currentDir.as_posix() + "/yaml/*.yaml"],
# A set of transformations to apply to Kubernetes resource definitions before registering with engine.
transformations=[virtHandlerNodePlacement],
)
pulumi/k3s-prod-1/visualization/virtual_machines/__init__.py
+1
View File
@@ -0,0 +1 @@
from .test import *
pulumi/k3s-prod-1/visualization/virtual_machines/test.py
+10
View File
@@ -0,0 +1,10 @@
from pulumi_kubernetes.yaml import ConfigGroup
from pathlib import Path
currentDir = Path(__file__).parent
kubevirt = ConfigGroup(
"virtual-machines",
files=[currentDir.as_posix() + "/yaml/*.yaml"],
)
pulumi/k3s-prod-1/visualization/virtual_machines/yaml/test.yaml
+34
View File
@@ -0,0 +1,34 @@
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
name: testvmi-nocloud
spec:
terminationGracePeriodSeconds: 30
domain:
resources:
requests:
memory: 1024M
devices:
disks:
- name: containerdisk
disk:
bus: virtio
- name: emptydisk
disk:
bus: virtio
- disk:
bus: virtio
name: cloudinitdisk
volumes:
- name: containerdisk
containerDisk:
image: kubevirt/fedora-cloud-container-disk-demo:latest
- name: emptydisk
emptyDisk:
capacity: "2Gi"
- name: cloudinitdisk
cloudInitNoCloud:
userData: |-
#cloud-config
password: fedora
chpasswd: { expire: False }
pulumi/k3s-prod-1/visualization/yaml/download.sh
+5
View File
@@ -0,0 +1,5 @@
export RELEASE=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt)
echo "The latest kubevirt's version is $RELEASE"
curl -Lo kubevirt-operator-${RELEASE}.yaml https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-operator.yaml
curl -Lo kubevirt-cr-${RELEASE}.yaml https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-cr.yaml
pulumi/k3s-prod-1/visualization/yaml/kubevirt-cr-v1.1.1.yaml
+14
View File
@@ -0,0 +1,14 @@
---
apiVersion: kubevirt.io/v1
kind: KubeVirt
metadata:
name: kubevirt
namespace: kubevirt
spec:
certificateRotateStrategy: {}
configuration:
developerConfiguration:
featureGates: []
customizeComponents: {}
imagePullPolicy: IfNotPresent
workloadUpdateStrategy: {}
pulumi/k3s-prod-1/visualization/yaml/kubevirt-operator-v1.1.1.yaml
+7529
View File
File diff suppressed because it is too large Load Diff