mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:36 +01:00
Merge pull request #314 from ryan4yin/feat/rustfs-s3
feat(youko): replace MinIO with RustFS S3
This commit is contained in:
28 files changed
+454
-288
No files matched your search
Generated
+20
-3
@@ -503,10 +503,10 @@
|
||||
"mysecrets": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1790263145,
|
||||
"narHash": "sha256-mD2038t9X7eyRGe2wn1jddhrVizc+SX8b43LoB5hEsc=",
|
||||
"lastModified": 1790320138,
|
||||
"narHash": "sha256-wx4fG9vS1Fde03PKYMyFNR+4PVaS2Hi1TqKnByAWjHQ=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "7e75aad771e8748f666a83832678f3fbb6378a73",
|
||||
"rev": "5063301c99e72244abba99cd5ae8456626ff42e0",
|
||||
"shallow": true,
|
||||
"type": "git",
|
||||
"url": "ssh://git@github.com/ryan4yin/nix-secrets.git"
|
||||
@@ -714,6 +714,22 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-small": {
|
||||
"locked": {
|
||||
"lastModified": 1790297907,
|
||||
"narHash": "sha256-3tsHNGgWZoi3D5Fbi6xvkjN97qHC92zpJAweyzPju2M=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a5e02ba8b823f3846c2336c7baad299a36a47cdf",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable-small",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-stable": {
|
||||
"locked": {
|
||||
"lastModified": 1789654592,
|
||||
@@ -907,6 +923,7 @@
|
||||
"nixpkgs-darwin": "nixpkgs-darwin",
|
||||
"nixpkgs-master": "nixpkgs-master",
|
||||
"nixpkgs-patched": "nixpkgs-patched",
|
||||
"nixpkgs-small": "nixpkgs-small",
|
||||
"nixpkgs-stable": "nixpkgs-stable",
|
||||
"nixvim": "nixvim",
|
||||
"nu_scripts": "nu_scripts",
|
||||
|
||||
@@ -49,6 +49,10 @@
|
||||
nixpkgs-patched.url = "github:ryan4yin/nixpkgs/nixos-unstable-patched";
|
||||
# get some latest packages from the master branch
|
||||
nixpkgs-master.url = "github:nixos/nixpkgs/master";
|
||||
# only for rustfs: nixos-unstable-small ships 1.0.0 (with the web console),
|
||||
# while the default unstable nixpkgs is still on 1.0.0-rc.6. Using a separate
|
||||
# input keeps rustfs current without bumping the whole system nixpkgs.
|
||||
nixpkgs-small.url = "github:nixos/nixpkgs/nixos-unstable-small";
|
||||
|
||||
# NixOS MicroVMs -- run the VMs directly on the hosts, replacing the former KubeVirt
|
||||
microvm = {
|
||||
|
||||
@@ -71,6 +71,35 @@ in
|
||||
respond @metrics 403
|
||||
reverse_proxy http://localhost:8000
|
||||
'';
|
||||
# RustFS replaced MinIO. SigV4 signs the Host header, so preserve it. No
|
||||
# `encode` here: compression would corrupt range/streaming responses.
|
||||
virtualHosts."s3.writefor.fun".extraConfig = ''
|
||||
${hostCommonConfig}
|
||||
reverse_proxy http://localhost:9000 {
|
||||
header_up Host {http.request.host}
|
||||
transport http {
|
||||
dial_timeout 300s
|
||||
read_timeout 300s
|
||||
write_timeout 300s
|
||||
}
|
||||
}
|
||||
'';
|
||||
# RustFS management console (websockets). The UI is served under
|
||||
# /rustfs/console/, so send the root there.
|
||||
virtualHosts."s3-console.writefor.fun".extraConfig = ''
|
||||
${hostCommonConfig}
|
||||
redir / /rustfs/console/ 302
|
||||
reverse_proxy http://localhost:9001 {
|
||||
header_up Host {http.request.host}
|
||||
header_up Upgrade {http.request.header.Upgrade}
|
||||
header_up Connection {http.request.header.Connection}
|
||||
transport http {
|
||||
dial_timeout 300s
|
||||
read_timeout 300s
|
||||
write_timeout 300s
|
||||
}
|
||||
}
|
||||
'';
|
||||
# Every other name is not a service on this host: answer 404 instead of
|
||||
# caddy's empty 200 fallback, which hides typos.
|
||||
virtualHosts."*.writefor.fun".extraConfig = ''
|
||||
@@ -83,6 +112,17 @@ in
|
||||
reverse_proxy http://localhost:9091
|
||||
'';
|
||||
|
||||
# In-cluster apps exposed through the k3s Istio gateway. That gateway is a
|
||||
# NodePort (:80) on every node; terminate TLS here and forward to the
|
||||
# workers, preserving the Host header so Istio routes by hostname. Grafana
|
||||
# (below) reaches Loki through this.
|
||||
virtualHosts."loki-gateway.writefor.fun".extraConfig = ''
|
||||
${hostCommonConfig}
|
||||
reverse_proxy http://192.168.5.111:80 http://192.168.5.112:80 http://192.168.5.113:80 {
|
||||
header_up Host {http.request.host}
|
||||
}
|
||||
'';
|
||||
|
||||
# Monitoring
|
||||
virtualHosts."uptime-kuma.writefor.fun".extraConfig = ''
|
||||
${hostCommonConfig}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
config,
|
||||
pkgs-small,
|
||||
...
|
||||
}:
|
||||
{
|
||||
# The service module comes from the default nixpkgs (services.rustfs);
|
||||
# pkgs-small (instantiated in outputs/default.nix) only supplies the newer
|
||||
# package. The module creates the rustfs user/group, the data volume via
|
||||
# tmpfiles, and a hardened systemd unit.
|
||||
services.rustfs = {
|
||||
enable = true;
|
||||
package = pkgs-small.rustfs;
|
||||
settings = {
|
||||
# New directory; the old MinIO tree at /data/apps/minio is kept as a
|
||||
# rollback snapshot and was carried over via an S3-level copy.
|
||||
RUSTFS_VOLUMES = "/data/apps/rustfs";
|
||||
# Loopback only; caddy terminates TLS and reverse-proxies 443 -> 9000/9001.
|
||||
RUSTFS_ADDRESS = "127.0.0.1:9000";
|
||||
RUSTFS_CONSOLE_ADDRESS = "127.0.0.1:9001";
|
||||
RUSTFS_CONSOLE_ENABLE = "true";
|
||||
# Sign/presign against the public hostname, since the public port (443)
|
||||
# differs from the internal one (9000).
|
||||
RUSTFS_SERVER_DOMAINS = "s3.writefor.fun";
|
||||
RUST_LOG = "info";
|
||||
};
|
||||
# Root credentials (access key = root "user", secret key = password), as a
|
||||
# dotenv secret with RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY.
|
||||
environmentFile = config.age.secrets."rustfs.env".path;
|
||||
};
|
||||
|
||||
# The data volume lives on the encrypted HDD subvolume; wait for its mount.
|
||||
systemd.services.rustfs.unitConfig.RequiresMountsFor = "/data/apps";
|
||||
}
|
||||
+8
-3
@@ -8,7 +8,8 @@ managing storage and backend services.
|
||||
```
|
||||
infra/
|
||||
├── README.md
|
||||
└── minio/ # MinIO S3-compatible storage configurations
|
||||
└── rustfs/ # RustFS S3-compatible storage (replaced MinIO)
|
||||
├── README.md # RustFS + rc runbook (buckets / IAM)
|
||||
├── loki/ # Loki log storage buckets
|
||||
│ ├── README.md
|
||||
│ ├── loki.tf # Loki-specific bucket configuration
|
||||
@@ -23,11 +24,15 @@ infra/
|
||||
|
||||
## Services Overview
|
||||
|
||||
### MinIO Storage
|
||||
### RustFS Storage
|
||||
|
||||
- **Loki Buckets**: Dedicated storage for Grafana Loki log aggregation
|
||||
- **Terraform Backend**: Centralized state management for all Terraform configurations
|
||||
|
||||
RustFS speaks the S3 API but not MinIO's Admin API, so buckets/lifecycle are managed with the AWS
|
||||
provider and IAM users/policies with the official `rc` client. See
|
||||
[rustfs/README.md](./rustfs/README.md).
|
||||
|
||||
### External Resources
|
||||
|
||||
- **Kubernetes YAML**: Managed in separate repository
|
||||
@@ -41,7 +46,7 @@ Each subdirectory contains its own Terraform configuration:
|
||||
1. **Navigate to specific service**:
|
||||
|
||||
```bash
|
||||
cd infra/minio/loki
|
||||
cd infra/rustfs/loki
|
||||
```
|
||||
|
||||
2. **Deploy configuration**:
|
||||
|
||||
Generated
-22
@@ -1,22 +0,0 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/aminueza/minio" {
|
||||
version = "3.5.2"
|
||||
constraints = "3.5.2"
|
||||
hashes = [
|
||||
"h1:3G/Q/dlf4ItE5tvE1zvSDUW4bYvwdCMVsHNAhMq9328=",
|
||||
"zh:5513c7b20eac89b7bc27b1f762ff03058b4c75456523d5065c41be170fc1ce53",
|
||||
"zh:597ec8ab8169ab4d044b7d442e65b03bbce2516c15f718510e8c80b5fc451be6",
|
||||
"zh:608ff0eb5929b840c11efee1da0273b81d21a8149d8f2d259989597068b48253",
|
||||
"zh:71bee58a6ba43d2a2aadd604c0e04f621fa67cb82ab3633fc5d1366689a5be6b",
|
||||
"zh:9871556bcc3d5daab3cd8e302d1d07bc5693038e1abf8bd11aaf07a439d67a0b",
|
||||
"zh:a3272fbb1ac7dff2481e778284709a5d8b85eda61f26239867eaed9ede57e90a",
|
||||
"zh:a5048a378d5b075a6afac14197fc0fc57f97788cd697749621c07cec7156344c",
|
||||
"zh:a8f28d070653cbd78ca85f9e54d9391a164828de598d481ed53d04882944dcb7",
|
||||
"zh:cbf6895d80828f66fdaa234c6fcf87c329c41eb72391a6d29056b917bce65426",
|
||||
"zh:cd48186b94cee7757a59f848dd6a2bd1d2faa76738a849261ca7cf14e7ca76c2",
|
||||
"zh:cdefdf9bb591ab19c3176c7c8796762e2626ebde0d49971b49393f6bf28533ba",
|
||||
"zh:ef16beff601be117a837cd47a1813be24ee0463d4f36a5d5f7e42a19d6c02b3d",
|
||||
]
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
# Buckets for Grafana Loki
|
||||
|
||||
Store Log data.
|
||||
@@ -1,84 +0,0 @@
|
||||
# ==============================================
|
||||
# Buckets
|
||||
# ==============================================
|
||||
|
||||
resource "minio_s3_bucket" "k3s-test-1-loki-chunks" {
|
||||
bucket = "k3s-test-1-loki-chunks"
|
||||
acl = "private"
|
||||
}
|
||||
|
||||
# ==============================================
|
||||
# Bucket Lifecycle
|
||||
# ==============================================
|
||||
|
||||
resource "minio_ilm_policy" "loki-chunks-expire-rules" {
|
||||
bucket = minio_s3_bucket.k3s-test-1-loki-chunks.bucket
|
||||
|
||||
rule {
|
||||
id = "expire-7d"
|
||||
status = "Enabled"
|
||||
expiration = "7d"
|
||||
}
|
||||
}
|
||||
|
||||
# ==============================================
|
||||
# User & Permission
|
||||
# ==============================================
|
||||
|
||||
|
||||
resource "minio_iam_user" "loki" {
|
||||
name = "loki"
|
||||
force_destroy = true
|
||||
tags = {
|
||||
env = "prod"
|
||||
managedBy = "terraform"
|
||||
}
|
||||
}
|
||||
|
||||
resource "minio_iam_service_account" "loki" {
|
||||
target_user = minio_iam_user.loki.name
|
||||
}
|
||||
|
||||
resource "minio_iam_policy" "loki" {
|
||||
name = "loki"
|
||||
policy = <<EOF
|
||||
{
|
||||
"Version":"2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "ObjectFullAccess",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": [
|
||||
"arn:aws:s3:::k3s-test-1-loki-chunks/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
resource "minio_iam_user_policy_attachment" "loki-1" {
|
||||
user_name = minio_iam_user.loki.id
|
||||
policy_name = minio_iam_policy.loki.id
|
||||
}
|
||||
|
||||
# ======================================================
|
||||
|
||||
output "loki-chunks_url" {
|
||||
value = minio_s3_bucket.k3s-test-1-loki-chunks.bucket_domain_name
|
||||
}
|
||||
|
||||
output "loki_accesskey" {
|
||||
value = minio_iam_service_account.loki.access_key
|
||||
}
|
||||
|
||||
output "loki_secretkey" {
|
||||
value = minio_iam_service_account.loki.secret_key
|
||||
sensitive = true
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
terraform {
|
||||
# https://developer.hashicorp.com/terraform/language/settings/backends/s3#credentials-and-shared-configuration
|
||||
backend "s3" {
|
||||
bucket = "tf-s3-backend"
|
||||
key = "homelab/minio/terraform.tfstate"
|
||||
region = "us-east-1"
|
||||
endpoints = {
|
||||
s3 = "https://minio.writefor.fun"
|
||||
}
|
||||
|
||||
# pass access key & secret via:
|
||||
# 1. env: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
|
||||
# 2. aws credential: ~/.aws/credentials
|
||||
# access_key = ""
|
||||
# secret_key = ""
|
||||
|
||||
# we're using minio, skip all aws related validation & checks
|
||||
skip_credentials_validation = true
|
||||
skip_metadata_api_check = true
|
||||
skip_region_validation = true
|
||||
skip_requesting_account_id = true
|
||||
use_path_style = true
|
||||
}
|
||||
|
||||
required_providers {
|
||||
minio = {
|
||||
source = "aminueza/minio"
|
||||
version = "3.5.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# https://registry.terraform.io/providers/aminueza/minio/latest/docs
|
||||
provider "minio" {
|
||||
minio_server = "minio.writefor.fun"
|
||||
minio_user = "ryan"
|
||||
|
||||
minio_api_version = "v4"
|
||||
minio_region = "us-east-1"
|
||||
minio_ssl = true
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
# for provider
|
||||
#
|
||||
# export MINIO_PASSWORD="xxx"
|
||||
|
||||
# for terraform's s3 backend
|
||||
#
|
||||
# export AWS_ACCESS_KEY_ID="xxx"
|
||||
# export AWS_SECRET_ACCESS_KEY="xxx"
|
||||
#
|
||||
terraform init
|
||||
terraform plan
|
||||
terraform apply
|
||||
|
||||
# show secret key
|
||||
terraform output loki_secretkey
|
||||
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/aminueza/minio" {
|
||||
version = "3.5.2"
|
||||
constraints = "3.5.2"
|
||||
hashes = [
|
||||
"h1:3G/Q/dlf4ItE5tvE1zvSDUW4bYvwdCMVsHNAhMq9328=",
|
||||
"zh:5513c7b20eac89b7bc27b1f762ff03058b4c75456523d5065c41be170fc1ce53",
|
||||
"zh:597ec8ab8169ab4d044b7d442e65b03bbce2516c15f718510e8c80b5fc451be6",
|
||||
"zh:608ff0eb5929b840c11efee1da0273b81d21a8149d8f2d259989597068b48253",
|
||||
"zh:71bee58a6ba43d2a2aadd604c0e04f621fa67cb82ab3633fc5d1366689a5be6b",
|
||||
"zh:9871556bcc3d5daab3cd8e302d1d07bc5693038e1abf8bd11aaf07a439d67a0b",
|
||||
"zh:a3272fbb1ac7dff2481e778284709a5d8b85eda61f26239867eaed9ede57e90a",
|
||||
"zh:a5048a378d5b075a6afac14197fc0fc57f97788cd697749621c07cec7156344c",
|
||||
"zh:a8f28d070653cbd78ca85f9e54d9391a164828de598d481ed53d04882944dcb7",
|
||||
"zh:cbf6895d80828f66fdaa234c6fcf87c329c41eb72391a6d29056b917bce65426",
|
||||
"zh:cd48186b94cee7757a59f848dd6a2bd1d2faa76738a849261ca7cf14e7ca76c2",
|
||||
"zh:cdefdf9bb591ab19c3176c7c8796762e2626ebde0d49971b49393f6bf28533ba",
|
||||
"zh:ef16beff601be117a837cd47a1813be24ee0463d4f36a5d5f7e42a19d6c02b3d",
|
||||
]
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
# Terraform's S3 Backend
|
||||
|
||||
This terraform workspace will be used only once, and we will not save the terraform.tfstate file.
|
||||
|
||||
It's used to create a minio bucket to store all other tfstate files.
|
||||
@@ -1,18 +0,0 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
minio = {
|
||||
source = "aminueza/minio"
|
||||
version = "3.5.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# https://registry.terraform.io/providers/aminueza/minio/latest/docs
|
||||
provider "minio" {
|
||||
minio_server = "minio.writefor.fun"
|
||||
minio_user = "ryan"
|
||||
|
||||
minio_api_version = "v4"
|
||||
minio_region = "us-east-1"
|
||||
minio_ssl = true
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
# for provider
|
||||
#
|
||||
# export MINIO_PASSWORD="xxx"
|
||||
#
|
||||
terraform init
|
||||
terraform plan
|
||||
terraform apply
|
||||
@@ -1,64 +0,0 @@
|
||||
# https://developer.hashicorp.com/terraform/language/settings/backends/s3
|
||||
resource "minio_s3_bucket" "tf-s3-backend" {
|
||||
bucket = "tf-s3-backend"
|
||||
acl = "private"
|
||||
}
|
||||
|
||||
resource "minio_iam_user" "tf-s3-backend" {
|
||||
name = "tf-s3-backend"
|
||||
force_destroy = true
|
||||
tags = {
|
||||
env = "prod"
|
||||
managedBy = "terraform"
|
||||
}
|
||||
}
|
||||
|
||||
resource "minio_iam_policy" "tf-s3-backend" {
|
||||
name = "tf-s3-backend"
|
||||
policy = <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:ListBucket",
|
||||
"Resource": "arn:aws:s3:::tf-s3-backend"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": "arn:aws:s3:::tf-s3-backend/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
resource "minio_iam_user_policy_attachment" "tf-s3-backend-1" {
|
||||
user_name = minio_iam_user.tf-s3-backend.id
|
||||
policy_name = minio_iam_policy.tf-s3-backend.id
|
||||
}
|
||||
|
||||
resource "minio_iam_service_account" "tf-s3-backend" {
|
||||
target_user = minio_iam_user.tf-s3-backend.name
|
||||
}
|
||||
|
||||
|
||||
# ======================================================
|
||||
|
||||
output "tf-s3-backend_id" {
|
||||
value = minio_s3_bucket.tf-s3-backend.id
|
||||
}
|
||||
|
||||
output "tf-s3-backend_url" {
|
||||
value = minio_s3_bucket.tf-s3-backend.bucket_domain_name
|
||||
}
|
||||
|
||||
output "tf-s3-backend_accesskey" {
|
||||
value = minio_iam_service_account.tf-s3-backend.access_key
|
||||
}
|
||||
|
||||
output "tf-s3-backend_secretkey" {
|
||||
value = minio_iam_service_account.tf-s3-backend.secret_key
|
||||
sensitive = true
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
# RustFS
|
||||
|
||||
RustFS replaced the retired MinIO at `s3.writefor.fun` (console: `s3-console.writefor.fun`). The
|
||||
service is declared in nix-config on the youko host; this directory manages its buckets/lifecycle
|
||||
and documents the IAM setup.
|
||||
|
||||
## Credentials
|
||||
|
||||
RustFS has no separate "root user" name: the root **access key** acts as the user and the **secret
|
||||
key** acts as the password. They live in agenix as a single dotenv secret; on youko it is decrypted
|
||||
to `/run/agenix/rustfs.env` (variables `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY`):
|
||||
|
||||
```bash
|
||||
set -a; . <(sudo cat /run/agenix/rustfs.env); set +a
|
||||
rc alias set rustfs https://s3.writefor.fun "$RUSTFS_ACCESS_KEY" "$RUSTFS_SECRET_KEY"
|
||||
```
|
||||
|
||||
## Buckets
|
||||
|
||||
```bash
|
||||
rc mb rustfs/tf-s3-backend
|
||||
rc mb rustfs/k3s-test-1-loki-chunks
|
||||
rc mb rustfs/k3s-test-1-loki-ruler
|
||||
rc mb rustfs/k3s-test-1-loki-admin
|
||||
|
||||
# 7-day expiry safety net on Loki chunks
|
||||
rc ilm rule add rustfs/k3s-test-1-loki-chunks --expiry-days 7
|
||||
```
|
||||
|
||||
If the buckets were carried over from MinIO (S3-level copy), they already exist; import them into
|
||||
Terraform instead (`terraform import`, see the workspace READMEs).
|
||||
|
||||
## IAM
|
||||
|
||||
### Loki user
|
||||
|
||||
`rc admin user add <alias>/ <accessKey> <secretKey>` creates a user whose access key is the given
|
||||
name. Generate a strong secret and reuse it when encrypting the k8s-gitops SOPS values.
|
||||
|
||||
```bash
|
||||
LOKI_SECRET="$(openssl rand -hex 20)"
|
||||
rc admin user add rustfs/ loki "$LOKI_SECRET"
|
||||
|
||||
cat > /tmp/loki-policy.json <<'EOF'
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "ObjectFullAccess",
|
||||
"Action": ["s3:PutObject", "s3:GetObject", "s3:ListBucket", "s3:DeleteObject"],
|
||||
"Effect": "Allow",
|
||||
"Resource": [
|
||||
"arn:aws:s3:::k3s-test-1-loki-chunks",
|
||||
"arn:aws:s3:::k3s-test-1-loki-chunks/*",
|
||||
"arn:aws:s3:::k3s-test-1-loki-ruler",
|
||||
"arn:aws:s3:::k3s-test-1-loki-ruler/*",
|
||||
"arn:aws:s3:::k3s-test-1-loki-admin",
|
||||
"arn:aws:s3:::k3s-test-1-loki-admin/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
rc admin policy create rustfs/ loki /tmp/loki-policy.json
|
||||
rc admin policy attach rustfs/ loki --user loki
|
||||
```
|
||||
|
||||
The k8s-gitops Loki values use access key `loki` and `$LOKI_SECRET`.
|
||||
|
||||
### Terraform credentials
|
||||
|
||||
Terraform runs with the RustFS root credentials; on youko export them before running it:
|
||||
|
||||
```bash
|
||||
set -a; . <(sudo cat /run/agenix/rustfs.env); set +a
|
||||
export AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY"
|
||||
```
|
||||
|
||||
Optionally, replace that with a dedicated account limited to the state bucket:
|
||||
|
||||
```bash
|
||||
TF_SECRET="$(openssl rand -hex 20)"
|
||||
rc admin user add rustfs/ tf-s3-backend "$TF_SECRET"
|
||||
|
||||
cat > /tmp/tf-s3-policy.json <<'EOF'
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:ListBucket",
|
||||
"Resource": "arn:aws:s3:::tf-s3-backend"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": "arn:aws:s3:::tf-s3-backend/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
rc admin policy create rustfs/ tf-s3-backend /tmp/tf-s3-policy.json
|
||||
rc admin policy attach rustfs/ tf-s3-backend --user tf-s3-backend
|
||||
```
|
||||
|
||||
Then export that account's keys as `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` instead.
|
||||
|
||||
## Carrying over the old MinIO data
|
||||
|
||||
The old tree is at `/data/apps/minio/data` on youko and is left untouched as a rollback snapshot.
|
||||
Serve it with a throwaway MinIO, then mirror with `rc`:
|
||||
|
||||
```bash
|
||||
podman run -d --rm --name minio-migrate \
|
||||
-p 127.0.0.1:19000:9000 \
|
||||
-v /data/apps/minio/data:/data \
|
||||
minio/minio server /data
|
||||
|
||||
rc alias set old http://127.0.0.1:19000 <temp-access> <temp-secret>
|
||||
rc mirror old/tf-s3-backend/ rustfs/tf-s3-backend/
|
||||
rc mirror old/k3s-test-1-loki-chunks/ rustfs/k3s-test-1-loki-chunks/
|
||||
|
||||
# verify, then remove the temp server
|
||||
podman rm -f minio-migrate
|
||||
```
|
||||
|
||||
Object data is not bound to MinIO's root credentials (only the encrypted IAM metadata in
|
||||
`.minio.sys` is), so a temporary credential works for the copy.
|
||||
Generated
+25
@@ -0,0 +1,25 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/aws" {
|
||||
version = "5.100.0"
|
||||
constraints = "~> 5.0"
|
||||
hashes = [
|
||||
"h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=",
|
||||
"zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644",
|
||||
"zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2",
|
||||
"zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274",
|
||||
"zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b",
|
||||
"zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862",
|
||||
"zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342",
|
||||
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||
"zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93",
|
||||
"zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2",
|
||||
"zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e",
|
||||
"zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421",
|
||||
"zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4",
|
||||
"zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9",
|
||||
"zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9",
|
||||
"zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70",
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# Buckets for Grafana Loki
|
||||
|
||||
Stores log data in RustFS (S3-compatible) at `s3.writefor.fun`.
|
||||
|
||||
Managed here: the `chunks`, `ruler`, and `admin` buckets.
|
||||
|
||||
Not managed here:
|
||||
|
||||
- The Loki IAM user and its policy. Create the user with the official `rc` client instead.
|
||||
- The 7-day expiry on `chunks`. The AWS provider's lifecycle resource waits on a response RustFS
|
||||
does not satisfy; use `rc ilm rule add rustfs/k3s-test-1-loki-chunks --expiry-days 7`.
|
||||
|
||||
See `../README.md` for both.
|
||||
@@ -0,0 +1,22 @@
|
||||
# ==============================================
|
||||
# Buckets
|
||||
# ==============================================
|
||||
|
||||
resource "aws_s3_bucket" "loki-chunks" {
|
||||
bucket = "k3s-test-1-loki-chunks"
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket" "loki-ruler" {
|
||||
bucket = "k3s-test-1-loki-ruler"
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket" "loki-admin" {
|
||||
bucket = "k3s-test-1-loki-admin"
|
||||
}
|
||||
|
||||
# NOTE: bucket lifecycle is NOT managed here. The AWS provider's
|
||||
# aws_s3_bucket_lifecycle_configuration waits on a GetBucketLifecycleConfiguration
|
||||
# response shape that RustFS does not satisfy (it times out). The 7-day expiry on
|
||||
# `chunks` is set with the official `rc` client instead:
|
||||
# rc ilm rule add rustfs/k3s-test-1-loki-chunks --expiry-days 7
|
||||
# (see ../README.md).
|
||||
@@ -0,0 +1,47 @@
|
||||
terraform {
|
||||
# https://developer.hashicorp.com/terraform/language/settings/backends/s3#credentials-and-shared-configuration
|
||||
backend "s3" {
|
||||
bucket = "tf-s3-backend"
|
||||
key = "homelab/rustfs/terraform.tfstate"
|
||||
region = "us-east-1"
|
||||
endpoints = {
|
||||
s3 = "https://s3.writefor.fun"
|
||||
}
|
||||
|
||||
# pass access key & secret via:
|
||||
# 1. env: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
|
||||
# 2. aws credential: ~/.aws/credentials
|
||||
# access_key = ""
|
||||
# secret_key = ""
|
||||
|
||||
# RustFS is S3-compatible but not AWS, so skip AWS-specific validation.
|
||||
skip_credentials_validation = true
|
||||
skip_metadata_api_check = true
|
||||
skip_region_validation = true
|
||||
skip_requesting_account_id = true
|
||||
use_path_style = true
|
||||
}
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Buckets/lifecycle go through the AWS provider pointed at the RustFS S3
|
||||
# endpoint; the Loki IAM user is created with `rc` (see ../README.md).
|
||||
provider "aws" {
|
||||
region = "us-east-1"
|
||||
|
||||
s3_use_path_style = true
|
||||
skip_credentials_validation = true
|
||||
skip_metadata_api_check = true
|
||||
skip_region_validation = true
|
||||
skip_requesting_account_id = true
|
||||
|
||||
endpoints {
|
||||
s3 = "https://s3.writefor.fun"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# Credentials for the RustFS S3 endpoint (used both for the S3 backend and the
|
||||
# AWS provider):
|
||||
#
|
||||
# export AWS_ACCESS_KEY_ID="xxx"
|
||||
# export AWS_SECRET_ACCESS_KEY="xxx"
|
||||
#
|
||||
# If the buckets already exist (carried over from MinIO by the S3-level copy),
|
||||
# import them first so terraform adopts them instead of failing:
|
||||
# terraform import aws_s3_bucket.loki-chunks k3s-test-1-loki-chunks
|
||||
# terraform import aws_s3_bucket.loki-ruler k3s-test-1-loki-ruler
|
||||
# terraform import aws_s3_bucket.loki-admin k3s-test-1-loki-admin
|
||||
#
|
||||
terraform init -reconfigure
|
||||
terraform plan
|
||||
terraform apply
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/aws" {
|
||||
version = "5.100.0"
|
||||
constraints = "~> 5.0"
|
||||
hashes = [
|
||||
"h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=",
|
||||
"zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644",
|
||||
"zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2",
|
||||
"zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274",
|
||||
"zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b",
|
||||
"zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862",
|
||||
"zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342",
|
||||
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||
"zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93",
|
||||
"zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2",
|
||||
"zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e",
|
||||
"zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421",
|
||||
"zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4",
|
||||
"zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9",
|
||||
"zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9",
|
||||
"zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70",
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
# Terraform's S3 Backend
|
||||
|
||||
This terraform workspace is used only once, and we do not save its `terraform.tfstate`.
|
||||
|
||||
It creates the `tf-s3-backend` RustFS bucket that stores all the other workspaces' tfstate files.
|
||||
|
||||
The IAM service account used to read/write that state is created with the official `rc` client, not
|
||||
here (RustFS does not implement MinIO's Admin API). See `../README.md` for the `rc` runbook.
|
||||
@@ -0,0 +1,26 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Buckets/lifecycle go through the AWS provider pointed at the RustFS S3
|
||||
# endpoint; IAM users/policies are created with `rc` (see ../README.md).
|
||||
#
|
||||
# Credentials come from the environment: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY.
|
||||
provider "aws" {
|
||||
region = "us-east-1"
|
||||
|
||||
s3_use_path_style = true
|
||||
skip_credentials_validation = true
|
||||
skip_metadata_api_check = true
|
||||
skip_region_validation = true
|
||||
skip_requesting_account_id = true
|
||||
|
||||
endpoints {
|
||||
s3 = "https://s3.writefor.fun"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# Credentials for the RustFS S3 endpoint: the RustFS root credentials, or a
|
||||
# dedicated account limited to this bucket (see ../README.md):
|
||||
#
|
||||
# export AWS_ACCESS_KEY_ID="xxx"
|
||||
# export AWS_SECRET_ACCESS_KEY="xxx"
|
||||
#
|
||||
# If the bucket already exists (carried over from MinIO by the S3-level copy),
|
||||
# import it first so terraform adopts it instead of failing to recreate it:
|
||||
# terraform import aws_s3_bucket.tf-s3-backend tf-s3-backend
|
||||
#
|
||||
terraform init
|
||||
terraform plan
|
||||
terraform apply
|
||||
@@ -0,0 +1,12 @@
|
||||
# https://developer.hashicorp.com/terraform/language/settings/backends/s3
|
||||
#
|
||||
# Only the bucket is managed here. The IAM service account that terraform uses
|
||||
# to read/write state is created with `rc` (see ../README.md); its keys are
|
||||
# exported as AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY before running terraform.
|
||||
resource "aws_s3_bucket" "tf-s3-backend" {
|
||||
bucket = "tf-s3-backend"
|
||||
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,12 @@ let
|
||||
# to use chrome, we need to allow the installation of non-free software
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
# only for rustfs: nixos-unstable-small ships rustfs 1.0.0 (with the web
|
||||
# console), while the default unstable nixpkgs is still 1.0.0-rc.6.
|
||||
pkgs-small = import inputs.nixpkgs-small {
|
||||
inherit system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
pkgs-blender = import inputs.nixpkgs-blender {
|
||||
inherit system;
|
||||
config = lib.optionalAttrs (system == "x86_64-linux") {
|
||||
|
||||
@@ -166,6 +166,14 @@ in
|
||||
mode = "0400";
|
||||
owner = "sftpgo";
|
||||
};
|
||||
|
||||
# RustFS root credentials as a systemd EnvironmentFile consumed by
|
||||
# services.rustfs: RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY.
|
||||
"rustfs.env" = {
|
||||
file = "${mysecrets}/server/rustfs.env.age";
|
||||
mode = "0400";
|
||||
owner = "root";
|
||||
};
|
||||
};
|
||||
})
|
||||
|
||||
|
||||
Reference in new issue
Block a user