From 7e0e4aa566f24d709b0a86b853360e7b6f380440 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Fri, 25 Sep 2026 14:21:18 +0800 Subject: [PATCH] feat(youko): replace MinIO with RustFS S3 Deploy RustFS in place of the retired MinIO and move the S3 consumers over. - flake.nix / flake.lock / outputs/default.nix: add a nixpkgs-small (nixos-unstable-small) input for the rustfs 1.0.0 package (with the web console), instantiated as pkgs-small in genSpecialArgs alongside the other nixpkgs instances; drop the rustfs flake input. The rest of the system stays on the default nixpkgs. - hosts/12kingdoms-youko/homelab-services/rustfs.nix: use nixpkgs' own services.rustfs module, with package = pkgs-small.rustfs and environmentFile = the agenix rustfs.env secret. - caddy.nix: s3.writefor.fun (S3 API) and s3-console.writefor.fun (console, redirects / to /rustfs/console/). - secrets/nixos.nix: declare a single rustfs.env (dotenv) secret holding RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY, replacing the separate key files. - infra/minio -> infra/rustfs: buckets are managed with the hashicorp/aws provider against the RustFS S3 endpoint; IAM/ILM are applied with rc. --- flake.lock | 23 +++- flake.nix | 4 + .../homelab-services/caddy.nix | 40 ++++++ .../homelab-services/rustfs.nix | 34 +++++ infra/README.md | 11 +- infra/minio/loki/.terraform.lock.hcl | 22 --- infra/minio/loki/README.md | 3 - infra/minio/loki/loki.tf | 84 ------------ infra/minio/loki/main.tf | 41 ------ infra/minio/loki/run.sh | 16 --- infra/minio/tf-s3-backend/.terraform.lock.hcl | 22 --- infra/minio/tf-s3-backend/README.md | 5 - infra/minio/tf-s3-backend/main.tf | 18 --- infra/minio/tf-s3-backend/run.sh | 7 - infra/minio/tf-s3-backend/tf-s3-backend.tf | 64 --------- infra/rustfs/README.md | 128 ++++++++++++++++++ infra/rustfs/loki/.terraform.lock.hcl | 25 ++++ infra/rustfs/loki/README.md | 13 ++ infra/rustfs/loki/loki.tf | 22 +++ infra/rustfs/loki/main.tf | 47 +++++++ infra/rustfs/loki/run.sh | 15 ++ .../rustfs/tf-s3-backend/.terraform.lock.hcl | 25 ++++ infra/rustfs/tf-s3-backend/README.md | 8 ++ infra/rustfs/tf-s3-backend/main.tf | 26 ++++ infra/rustfs/tf-s3-backend/run.sh | 13 ++ infra/rustfs/tf-s3-backend/tf-s3-backend.tf | 12 ++ outputs/default.nix | 6 + secrets/nixos.nix | 8 ++ 28 files changed, 454 insertions(+), 288 deletions(-) create mode 100644 hosts/12kingdoms-youko/homelab-services/rustfs.nix delete mode 100644 infra/minio/loki/.terraform.lock.hcl delete mode 100644 infra/minio/loki/README.md delete mode 100644 infra/minio/loki/loki.tf delete mode 100644 infra/minio/loki/main.tf delete mode 100644 infra/minio/loki/run.sh delete mode 100644 infra/minio/tf-s3-backend/.terraform.lock.hcl delete mode 100644 infra/minio/tf-s3-backend/README.md delete mode 100644 infra/minio/tf-s3-backend/main.tf delete mode 100644 infra/minio/tf-s3-backend/run.sh delete mode 100644 infra/minio/tf-s3-backend/tf-s3-backend.tf create mode 100644 infra/rustfs/README.md create mode 100644 infra/rustfs/loki/.terraform.lock.hcl create mode 100644 infra/rustfs/loki/README.md create mode 100644 infra/rustfs/loki/loki.tf create mode 100644 infra/rustfs/loki/main.tf create mode 100644 infra/rustfs/loki/run.sh create mode 100644 infra/rustfs/tf-s3-backend/.terraform.lock.hcl create mode 100644 infra/rustfs/tf-s3-backend/README.md create mode 100644 infra/rustfs/tf-s3-backend/main.tf create mode 100644 infra/rustfs/tf-s3-backend/run.sh create mode 100644 infra/rustfs/tf-s3-backend/tf-s3-backend.tf diff --git a/flake.lock b/flake.lock index 7b2d8a52..af7cb5d6 100644 --- a/flake.lock +++ b/flake.lock @@ -503,10 +503,10 @@ "mysecrets": { "flake": false, "locked": { - "lastModified": 1790263145, - "narHash": "sha256-mD2038t9X7eyRGe2wn1jddhrVizc+SX8b43LoB5hEsc=", + "lastModified": 1790320138, + "narHash": "sha256-wx4fG9vS1Fde03PKYMyFNR+4PVaS2Hi1TqKnByAWjHQ=", "ref": "refs/heads/main", - "rev": "7e75aad771e8748f666a83832678f3fbb6378a73", + "rev": "5063301c99e72244abba99cd5ae8456626ff42e0", "shallow": true, "type": "git", "url": "ssh://git@github.com/ryan4yin/nix-secrets.git" @@ -714,6 +714,22 @@ "type": "github" } }, + "nixpkgs-small": { + "locked": { + "lastModified": 1790297907, + "narHash": "sha256-3tsHNGgWZoi3D5Fbi6xvkjN97qHC92zpJAweyzPju2M=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a5e02ba8b823f3846c2336c7baad299a36a47cdf", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-stable": { "locked": { "lastModified": 1789654592, @@ -907,6 +923,7 @@ "nixpkgs-darwin": "nixpkgs-darwin", "nixpkgs-master": "nixpkgs-master", "nixpkgs-patched": "nixpkgs-patched", + "nixpkgs-small": "nixpkgs-small", "nixpkgs-stable": "nixpkgs-stable", "nixvim": "nixvim", "nu_scripts": "nu_scripts", diff --git a/flake.nix b/flake.nix index 36647037..280b6c45 100644 --- a/flake.nix +++ b/flake.nix @@ -49,6 +49,10 @@ nixpkgs-patched.url = "github:ryan4yin/nixpkgs/nixos-unstable-patched"; # get some latest packages from the master branch nixpkgs-master.url = "github:nixos/nixpkgs/master"; + # only for rustfs: nixos-unstable-small ships 1.0.0 (with the web console), + # while the default unstable nixpkgs is still on 1.0.0-rc.6. Using a separate + # input keeps rustfs current without bumping the whole system nixpkgs. + nixpkgs-small.url = "github:nixos/nixpkgs/nixos-unstable-small"; # NixOS MicroVMs -- run the VMs directly on the hosts, replacing the former KubeVirt microvm = { diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 1b500dfb..27244d82 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -71,6 +71,35 @@ in respond @metrics 403 reverse_proxy http://localhost:8000 ''; + # RustFS replaced MinIO. SigV4 signs the Host header, so preserve it. No + # `encode` here: compression would corrupt range/streaming responses. + virtualHosts."s3.writefor.fun".extraConfig = '' + ${hostCommonConfig} + reverse_proxy http://localhost:9000 { + header_up Host {http.request.host} + transport http { + dial_timeout 300s + read_timeout 300s + write_timeout 300s + } + } + ''; + # RustFS management console (websockets). The UI is served under + # /rustfs/console/, so send the root there. + virtualHosts."s3-console.writefor.fun".extraConfig = '' + ${hostCommonConfig} + redir / /rustfs/console/ 302 + reverse_proxy http://localhost:9001 { + header_up Host {http.request.host} + header_up Upgrade {http.request.header.Upgrade} + header_up Connection {http.request.header.Connection} + transport http { + dial_timeout 300s + read_timeout 300s + write_timeout 300s + } + } + ''; # Every other name is not a service on this host: answer 404 instead of # caddy's empty 200 fallback, which hides typos. virtualHosts."*.writefor.fun".extraConfig = '' @@ -83,6 +112,17 @@ in reverse_proxy http://localhost:9091 ''; + # In-cluster apps exposed through the k3s Istio gateway. That gateway is a + # NodePort (:80) on every node; terminate TLS here and forward to the + # workers, preserving the Host header so Istio routes by hostname. Grafana + # (below) reaches Loki through this. + virtualHosts."loki-gateway.writefor.fun".extraConfig = '' + ${hostCommonConfig} + reverse_proxy http://192.168.5.111:80 http://192.168.5.112:80 http://192.168.5.113:80 { + header_up Host {http.request.host} + } + ''; + # Monitoring virtualHosts."uptime-kuma.writefor.fun".extraConfig = '' ${hostCommonConfig} diff --git a/hosts/12kingdoms-youko/homelab-services/rustfs.nix b/hosts/12kingdoms-youko/homelab-services/rustfs.nix new file mode 100644 index 00000000..55795f72 --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/rustfs.nix @@ -0,0 +1,34 @@ +{ + config, + pkgs-small, + ... +}: +{ + # The service module comes from the default nixpkgs (services.rustfs); + # pkgs-small (instantiated in outputs/default.nix) only supplies the newer + # package. The module creates the rustfs user/group, the data volume via + # tmpfiles, and a hardened systemd unit. + services.rustfs = { + enable = true; + package = pkgs-small.rustfs; + settings = { + # New directory; the old MinIO tree at /data/apps/minio is kept as a + # rollback snapshot and was carried over via an S3-level copy. + RUSTFS_VOLUMES = "/data/apps/rustfs"; + # Loopback only; caddy terminates TLS and reverse-proxies 443 -> 9000/9001. + RUSTFS_ADDRESS = "127.0.0.1:9000"; + RUSTFS_CONSOLE_ADDRESS = "127.0.0.1:9001"; + RUSTFS_CONSOLE_ENABLE = "true"; + # Sign/presign against the public hostname, since the public port (443) + # differs from the internal one (9000). + RUSTFS_SERVER_DOMAINS = "s3.writefor.fun"; + RUST_LOG = "info"; + }; + # Root credentials (access key = root "user", secret key = password), as a + # dotenv secret with RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY. + environmentFile = config.age.secrets."rustfs.env".path; + }; + + # The data volume lives on the encrypted HDD subvolume; wait for its mount. + systemd.services.rustfs.unitConfig.RequiresMountsFor = "/data/apps"; +} diff --git a/infra/README.md b/infra/README.md index 74efad98..0e6ae0f7 100644 --- a/infra/README.md +++ b/infra/README.md @@ -8,7 +8,8 @@ managing storage and backend services. ``` infra/ ├── README.md -└── minio/ # MinIO S3-compatible storage configurations +└── rustfs/ # RustFS S3-compatible storage (replaced MinIO) + ├── README.md # RustFS + rc runbook (buckets / IAM) ├── loki/ # Loki log storage buckets │ ├── README.md │ ├── loki.tf # Loki-specific bucket configuration @@ -23,11 +24,15 @@ infra/ ## Services Overview -### MinIO Storage +### RustFS Storage - **Loki Buckets**: Dedicated storage for Grafana Loki log aggregation - **Terraform Backend**: Centralized state management for all Terraform configurations +RustFS speaks the S3 API but not MinIO's Admin API, so buckets/lifecycle are managed with the AWS +provider and IAM users/policies with the official `rc` client. See +[rustfs/README.md](./rustfs/README.md). + ### External Resources - **Kubernetes YAML**: Managed in separate repository @@ -41,7 +46,7 @@ Each subdirectory contains its own Terraform configuration: 1. **Navigate to specific service**: ```bash - cd infra/minio/loki + cd infra/rustfs/loki ``` 2. **Deploy configuration**: diff --git a/infra/minio/loki/.terraform.lock.hcl b/infra/minio/loki/.terraform.lock.hcl deleted file mode 100644 index a7c7f027..00000000 --- a/infra/minio/loki/.terraform.lock.hcl +++ /dev/null @@ -1,22 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/aminueza/minio" { - version = "3.5.2" - constraints = "3.5.2" - hashes = [ - "h1:3G/Q/dlf4ItE5tvE1zvSDUW4bYvwdCMVsHNAhMq9328=", - "zh:5513c7b20eac89b7bc27b1f762ff03058b4c75456523d5065c41be170fc1ce53", - "zh:597ec8ab8169ab4d044b7d442e65b03bbce2516c15f718510e8c80b5fc451be6", - "zh:608ff0eb5929b840c11efee1da0273b81d21a8149d8f2d259989597068b48253", - "zh:71bee58a6ba43d2a2aadd604c0e04f621fa67cb82ab3633fc5d1366689a5be6b", - "zh:9871556bcc3d5daab3cd8e302d1d07bc5693038e1abf8bd11aaf07a439d67a0b", - "zh:a3272fbb1ac7dff2481e778284709a5d8b85eda61f26239867eaed9ede57e90a", - "zh:a5048a378d5b075a6afac14197fc0fc57f97788cd697749621c07cec7156344c", - "zh:a8f28d070653cbd78ca85f9e54d9391a164828de598d481ed53d04882944dcb7", - "zh:cbf6895d80828f66fdaa234c6fcf87c329c41eb72391a6d29056b917bce65426", - "zh:cd48186b94cee7757a59f848dd6a2bd1d2faa76738a849261ca7cf14e7ca76c2", - "zh:cdefdf9bb591ab19c3176c7c8796762e2626ebde0d49971b49393f6bf28533ba", - "zh:ef16beff601be117a837cd47a1813be24ee0463d4f36a5d5f7e42a19d6c02b3d", - ] -} diff --git a/infra/minio/loki/README.md b/infra/minio/loki/README.md deleted file mode 100644 index ab653fe7..00000000 --- a/infra/minio/loki/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Buckets for Grafana Loki - -Store Log data. diff --git a/infra/minio/loki/loki.tf b/infra/minio/loki/loki.tf deleted file mode 100644 index c7eae8be..00000000 --- a/infra/minio/loki/loki.tf +++ /dev/null @@ -1,84 +0,0 @@ -# ============================================== -# Buckets -# ============================================== - -resource "minio_s3_bucket" "k3s-test-1-loki-chunks" { - bucket = "k3s-test-1-loki-chunks" - acl = "private" -} - -# ============================================== -# Bucket Lifecycle -# ============================================== - -resource "minio_ilm_policy" "loki-chunks-expire-rules" { - bucket = minio_s3_bucket.k3s-test-1-loki-chunks.bucket - - rule { - id = "expire-7d" - status = "Enabled" - expiration = "7d" - } -} - -# ============================================== -# User & Permission -# ============================================== - - -resource "minio_iam_user" "loki" { - name = "loki" - force_destroy = true - tags = { - env = "prod" - managedBy = "terraform" - } -} - -resource "minio_iam_service_account" "loki" { - target_user = minio_iam_user.loki.name -} - -resource "minio_iam_policy" "loki" { - name = "loki" - policy = </ ` creates a user whose access key is the given +name. Generate a strong secret and reuse it when encrypting the k8s-gitops SOPS values. + +```bash +LOKI_SECRET="$(openssl rand -hex 20)" +rc admin user add rustfs/ loki "$LOKI_SECRET" + +cat > /tmp/loki-policy.json <<'EOF' +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ObjectFullAccess", + "Action": ["s3:PutObject", "s3:GetObject", "s3:ListBucket", "s3:DeleteObject"], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::k3s-test-1-loki-chunks", + "arn:aws:s3:::k3s-test-1-loki-chunks/*", + "arn:aws:s3:::k3s-test-1-loki-ruler", + "arn:aws:s3:::k3s-test-1-loki-ruler/*", + "arn:aws:s3:::k3s-test-1-loki-admin", + "arn:aws:s3:::k3s-test-1-loki-admin/*" + ] + } + ] +} +EOF +rc admin policy create rustfs/ loki /tmp/loki-policy.json +rc admin policy attach rustfs/ loki --user loki +``` + +The k8s-gitops Loki values use access key `loki` and `$LOKI_SECRET`. + +### Terraform credentials + +Terraform runs with the RustFS root credentials; on youko export them before running it: + +```bash +set -a; . <(sudo cat /run/agenix/rustfs.env); set +a +export AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" +``` + +Optionally, replace that with a dedicated account limited to the state bucket: + +```bash +TF_SECRET="$(openssl rand -hex 20)" +rc admin user add rustfs/ tf-s3-backend "$TF_SECRET" + +cat > /tmp/tf-s3-policy.json <<'EOF' +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::tf-s3-backend" + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": "arn:aws:s3:::tf-s3-backend/*" + } + ] +} +EOF +rc admin policy create rustfs/ tf-s3-backend /tmp/tf-s3-policy.json +rc admin policy attach rustfs/ tf-s3-backend --user tf-s3-backend +``` + +Then export that account's keys as `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` instead. + +## Carrying over the old MinIO data + +The old tree is at `/data/apps/minio/data` on youko and is left untouched as a rollback snapshot. +Serve it with a throwaway MinIO, then mirror with `rc`: + +```bash +podman run -d --rm --name minio-migrate \ + -p 127.0.0.1:19000:9000 \ + -v /data/apps/minio/data:/data \ + minio/minio server /data + +rc alias set old http://127.0.0.1:19000 +rc mirror old/tf-s3-backend/ rustfs/tf-s3-backend/ +rc mirror old/k3s-test-1-loki-chunks/ rustfs/k3s-test-1-loki-chunks/ + +# verify, then remove the temp server +podman rm -f minio-migrate +``` + +Object data is not bound to MinIO's root credentials (only the encrypted IAM metadata in +`.minio.sys` is), so a temporary credential works for the copy. diff --git a/infra/rustfs/loki/.terraform.lock.hcl b/infra/rustfs/loki/.terraform.lock.hcl new file mode 100644 index 00000000..39a6fbd3 --- /dev/null +++ b/infra/rustfs/loki/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} diff --git a/infra/rustfs/loki/README.md b/infra/rustfs/loki/README.md new file mode 100644 index 00000000..fe6b1dfc --- /dev/null +++ b/infra/rustfs/loki/README.md @@ -0,0 +1,13 @@ +# Buckets for Grafana Loki + +Stores log data in RustFS (S3-compatible) at `s3.writefor.fun`. + +Managed here: the `chunks`, `ruler`, and `admin` buckets. + +Not managed here: + +- The Loki IAM user and its policy. Create the user with the official `rc` client instead. +- The 7-day expiry on `chunks`. The AWS provider's lifecycle resource waits on a response RustFS + does not satisfy; use `rc ilm rule add rustfs/k3s-test-1-loki-chunks --expiry-days 7`. + +See `../README.md` for both. diff --git a/infra/rustfs/loki/loki.tf b/infra/rustfs/loki/loki.tf new file mode 100644 index 00000000..3da71119 --- /dev/null +++ b/infra/rustfs/loki/loki.tf @@ -0,0 +1,22 @@ +# ============================================== +# Buckets +# ============================================== + +resource "aws_s3_bucket" "loki-chunks" { + bucket = "k3s-test-1-loki-chunks" +} + +resource "aws_s3_bucket" "loki-ruler" { + bucket = "k3s-test-1-loki-ruler" +} + +resource "aws_s3_bucket" "loki-admin" { + bucket = "k3s-test-1-loki-admin" +} + +# NOTE: bucket lifecycle is NOT managed here. The AWS provider's +# aws_s3_bucket_lifecycle_configuration waits on a GetBucketLifecycleConfiguration +# response shape that RustFS does not satisfy (it times out). The 7-day expiry on +# `chunks` is set with the official `rc` client instead: +# rc ilm rule add rustfs/k3s-test-1-loki-chunks --expiry-days 7 +# (see ../README.md). diff --git a/infra/rustfs/loki/main.tf b/infra/rustfs/loki/main.tf new file mode 100644 index 00000000..75bfbbdc --- /dev/null +++ b/infra/rustfs/loki/main.tf @@ -0,0 +1,47 @@ +terraform { + # https://developer.hashicorp.com/terraform/language/settings/backends/s3#credentials-and-shared-configuration + backend "s3" { + bucket = "tf-s3-backend" + key = "homelab/rustfs/terraform.tfstate" + region = "us-east-1" + endpoints = { + s3 = "https://s3.writefor.fun" + } + + # pass access key & secret via: + # 1. env: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + # 2. aws credential: ~/.aws/credentials + # access_key = "" + # secret_key = "" + + # RustFS is S3-compatible but not AWS, so skip AWS-specific validation. + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + use_path_style = true + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +# Buckets/lifecycle go through the AWS provider pointed at the RustFS S3 +# endpoint; the Loki IAM user is created with `rc` (see ../README.md). +provider "aws" { + region = "us-east-1" + + s3_use_path_style = true + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + + endpoints { + s3 = "https://s3.writefor.fun" + } +} diff --git a/infra/rustfs/loki/run.sh b/infra/rustfs/loki/run.sh new file mode 100644 index 00000000..93ee85d7 --- /dev/null +++ b/infra/rustfs/loki/run.sh @@ -0,0 +1,15 @@ +# Credentials for the RustFS S3 endpoint (used both for the S3 backend and the +# AWS provider): +# +# export AWS_ACCESS_KEY_ID="xxx" +# export AWS_SECRET_ACCESS_KEY="xxx" +# +# If the buckets already exist (carried over from MinIO by the S3-level copy), +# import them first so terraform adopts them instead of failing: +# terraform import aws_s3_bucket.loki-chunks k3s-test-1-loki-chunks +# terraform import aws_s3_bucket.loki-ruler k3s-test-1-loki-ruler +# terraform import aws_s3_bucket.loki-admin k3s-test-1-loki-admin +# +terraform init -reconfigure +terraform plan +terraform apply diff --git a/infra/rustfs/tf-s3-backend/.terraform.lock.hcl b/infra/rustfs/tf-s3-backend/.terraform.lock.hcl new file mode 100644 index 00000000..39a6fbd3 --- /dev/null +++ b/infra/rustfs/tf-s3-backend/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} diff --git a/infra/rustfs/tf-s3-backend/README.md b/infra/rustfs/tf-s3-backend/README.md new file mode 100644 index 00000000..d906671e --- /dev/null +++ b/infra/rustfs/tf-s3-backend/README.md @@ -0,0 +1,8 @@ +# Terraform's S3 Backend + +This terraform workspace is used only once, and we do not save its `terraform.tfstate`. + +It creates the `tf-s3-backend` RustFS bucket that stores all the other workspaces' tfstate files. + +The IAM service account used to read/write that state is created with the official `rc` client, not +here (RustFS does not implement MinIO's Admin API). See `../README.md` for the `rc` runbook. diff --git a/infra/rustfs/tf-s3-backend/main.tf b/infra/rustfs/tf-s3-backend/main.tf new file mode 100644 index 00000000..c22093f4 --- /dev/null +++ b/infra/rustfs/tf-s3-backend/main.tf @@ -0,0 +1,26 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +# Buckets/lifecycle go through the AWS provider pointed at the RustFS S3 +# endpoint; IAM users/policies are created with `rc` (see ../README.md). +# +# Credentials come from the environment: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY. +provider "aws" { + region = "us-east-1" + + s3_use_path_style = true + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + + endpoints { + s3 = "https://s3.writefor.fun" + } +} diff --git a/infra/rustfs/tf-s3-backend/run.sh b/infra/rustfs/tf-s3-backend/run.sh new file mode 100644 index 00000000..a6c73cff --- /dev/null +++ b/infra/rustfs/tf-s3-backend/run.sh @@ -0,0 +1,13 @@ +# Credentials for the RustFS S3 endpoint: the RustFS root credentials, or a +# dedicated account limited to this bucket (see ../README.md): +# +# export AWS_ACCESS_KEY_ID="xxx" +# export AWS_SECRET_ACCESS_KEY="xxx" +# +# If the bucket already exists (carried over from MinIO by the S3-level copy), +# import it first so terraform adopts it instead of failing to recreate it: +# terraform import aws_s3_bucket.tf-s3-backend tf-s3-backend +# +terraform init +terraform plan +terraform apply diff --git a/infra/rustfs/tf-s3-backend/tf-s3-backend.tf b/infra/rustfs/tf-s3-backend/tf-s3-backend.tf new file mode 100644 index 00000000..84db5a36 --- /dev/null +++ b/infra/rustfs/tf-s3-backend/tf-s3-backend.tf @@ -0,0 +1,12 @@ +# https://developer.hashicorp.com/terraform/language/settings/backends/s3 +# +# Only the bucket is managed here. The IAM service account that terraform uses +# to read/write state is created with `rc` (see ../README.md); its keys are +# exported as AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY before running terraform. +resource "aws_s3_bucket" "tf-s3-backend" { + bucket = "tf-s3-backend" + + lifecycle { + prevent_destroy = true + } +} diff --git a/outputs/default.nix b/outputs/default.nix index 3dd4d445..48ab397f 100644 --- a/outputs/default.nix +++ b/outputs/default.nix @@ -69,6 +69,12 @@ let # to use chrome, we need to allow the installation of non-free software config.allowUnfree = true; }; + # only for rustfs: nixos-unstable-small ships rustfs 1.0.0 (with the web + # console), while the default unstable nixpkgs is still 1.0.0-rc.6. + pkgs-small = import inputs.nixpkgs-small { + inherit system; + config.allowUnfree = true; + }; pkgs-blender = import inputs.nixpkgs-blender { inherit system; config = lib.optionalAttrs (system == "x86_64-linux") { diff --git a/secrets/nixos.nix b/secrets/nixos.nix index 5a284e0d..20497b03 100644 --- a/secrets/nixos.nix +++ b/secrets/nixos.nix @@ -166,6 +166,14 @@ in mode = "0400"; owner = "sftpgo"; }; + + # RustFS root credentials as a systemd EnvironmentFile consumed by + # services.rustfs: RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY. + "rustfs.env" = { + file = "${mysecrets}/server/rustfs.env.age"; + mode = "0400"; + owner = "root"; + }; }; })