diff --git a/hosts/idols-aquamarine/caddy.nix b/hosts/idols-aquamarine/caddy.nix
index f30a9e7e..38dccbe6 100644
--- a/hosts/idols-aquamarine/caddy.nix
+++ b/hosts/idols-aquamarine/caddy.nix
@@ -127,6 +127,7 @@ in {
   networking.firewall.allowedTCPPorts = [80 443];
 
   # Create Directories
+  # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
     "d /data/apps/caddy/fileserver/ 0755 caddy caddy"
     # directory for virtual machine's images
diff --git a/hosts/idols-aquamarine/oci-containers/homepage/config/services.yaml b/hosts/idols-aquamarine/oci-containers/homepage/config/services.yaml
index 65588d32..f49ab249 100644
--- a/hosts/idols-aquamarine/oci-containers/homepage/config/services.yaml
+++ b/hosts/idols-aquamarine/oci-containers/homepage/config/services.yaml
@@ -26,12 +26,14 @@
     - LongHorn-Storage:
         icon: longhorn.svg
         href: http://longhorn.writefor.fun/
-    - Victoria-Metrics:
-        icon: si-victoriametrics
-        href: http://vm.writefor.fun/
-    - KubeVirt-Grafana:
-        icon: grafana.svg
-        href: http://k8s-grafana.writefor.fun/
+
+    # remote write to main prometheus, disable those two
+    # - Victoria-Metrics:
+    #     icon: si-victoriametrics
+    #     href: http://vm.writefor.fun/
+    # - KubeVirt-Grafana:
+    #     icon: grafana.svg
+    #     href: http://k8s-grafana.writefor.fun/
 
 - Homelab Monitoring:
     - Grafana:
@@ -56,7 +58,12 @@
         href: "https://sftpgo.writefor.fun/web/admin/folders"
         description: WebDAV & SFTP server
         siteMonitor: https://sftpgo.writefor.fun/
-#
+    - MinIO:
+        icon: minio.png
+        href: "https://minio-ui.writefor.fun/"
+        description: S3 compatible object storage
+        siteMonitor: "https://minio-ui.writefor.fun/"
+
 # - Kubernetes Monitoring:
 #     # TODO: Update this
 #     - Emby:
diff --git a/hosts/idols-aquamarine/oci-containers/uptime-kuma/default.nix b/hosts/idols-aquamarine/oci-containers/uptime-kuma/default.nix
index 98c360f2..d7acdead 100644
--- a/hosts/idols-aquamarine/oci-containers/uptime-kuma/default.nix
+++ b/hosts/idols-aquamarine/oci-containers/uptime-kuma/default.nix
@@ -10,6 +10,7 @@ in {
   };
 
   # Create Directories
+  # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
     "d ${dataDir} 0755 ${user} ${user}"
   ];
diff --git a/hosts/idols-aquamarine/postgresql.nix b/hosts/idols-aquamarine/postgresql.nix
index f0c89b0e..822a36b6 100644
--- a/hosts/idols-aquamarine/postgresql.nix
+++ b/hosts/idols-aquamarine/postgresql.nix
@@ -12,7 +12,9 @@
   dataDir = "/data/apps/postgresql/${package.psqlSchema}";
 in {
   # Create Directories
+  # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
+    "d /data/apps/postgresql 0755 ${user} ${user}"
     "d ${dataDir} 0755 ${user} ${user}"
   ];
 
@@ -32,12 +34,18 @@ in {
     # Ensures that the specified databases exist.
     ensureDatabases = [
       "mytestdb" # for testing
-      "openobserve"
       "juicefs"
+      # openobserve for every k8s clusters
+      "o2_k3s_test_1"
+      "o2_k3s_prod_1"
     ];
     ensureUsers = [
       {
-        name = "openobserve";
+        name = "o2_k3s_test_1";
+        ensureDBOwnership = true;
+      }
+      {
+        name = "o2_k3s_prod_1";
         ensureDBOwnership = true;
       }
       {
diff --git a/hosts/idols-aquamarine/prometheus/default.nix b/hosts/idols-aquamarine/prometheus/default.nix
index 1db049ae..9ba1a7c3 100644
--- a/hosts/idols-aquamarine/prometheus/default.nix
+++ b/hosts/idols-aquamarine/prometheus/default.nix
@@ -5,6 +5,7 @@
   ...
 }: {
   # Workaround for prometheus to store data in another place
+  # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
     "D /data/apps/prometheus2 0751 prometheus prometheus - -"
     "L+ /var/lib/prometheus2 - - - - /data/apps/prometheus2"
diff --git a/hosts/idols-aquamarine/sftpgo.nix b/hosts/idols-aquamarine/sftpgo.nix
index 7638bce5..d1350ae6 100644
--- a/hosts/idols-aquamarine/sftpgo.nix
+++ b/hosts/idols-aquamarine/sftpgo.nix
@@ -6,6 +6,7 @@ in {
   systemd.services.sftpgo.serviceConfig.EnvironmentFile = config.age.secrets."sftpgo.env".path;
 
   # Create Directories
+  # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
     "d ${dataDir} 0755 ${user} ${user}"
   ];
diff --git a/infra/.gitignore b/infra/.gitignore
new file mode 100644
index 00000000..a73931c9
--- /dev/null
+++ b/infra/.gitignore
@@ -0,0 +1,37 @@
+# Local .terraform directories
+**/.terraform/
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore transient lock info files created by terraform apply
+.terraform.tfstate.lock.info
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
diff --git a/infra/README.md b/infra/README.md
new file mode 100644
index 00000000..ddce864b
--- /dev/null
+++ b/infra/README.md
@@ -0,0 +1,6 @@
+# Infrastructure as Code
+
+Home for my infra-as-code.
+
+Kubernetes's yaml are stored in a seperate repo:
+[ryan4yin/k8s-gitops](https://github.com/ryan4yin/k8s-gitops).
diff --git a/infra/minio/openobserve/.terraform.lock.hcl b/infra/minio/openobserve/.terraform.lock.hcl
new file mode 100644
index 00000000..2579c2d3
--- /dev/null
+++ b/infra/minio/openobserve/.terraform.lock.hcl
@@ -0,0 +1,22 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/aminueza/minio" {
+  version     = "2.5.0"
+  constraints = "2.5.0"
+  hashes = [
+    "h1:RrjfsRy+fBVh7VF3r9u7uCCSjAdR5APa6sqbc9b8GfU=",
+    "zh:066cdb289dbfd1675e22fe58c8b42e2732f24fc1528b1919a78dfe28f80e8b30",
+    "zh:26d5e55106259e69493b95058178ec3d6b2395f03a8fe832af1be0e4d89ef42c",
+    "zh:6247e19de9ec6ef719cfcb174b8f08085c0fd5118b3b0de3fb9bb150702b4ad8",
+    "zh:70c3cbab0ba8edeec0db2e175bcdb47255c92f3153f839c4e8f2b0fe8c1366f4",
+    "zh:713793b4b93ae62070b18983ff525390de6c84547cab4220aa068437149f5035",
+    "zh:72de3e532d4bc7c7a4a872aaf00d7e4dfa09f3730668a738bb881d6734248f02",
+    "zh:9090f9288d7bc9f23043c1e65d8535e91f10413a16699d4a18add811b25fa167",
+    "zh:9847284aecb52718468feccb914d67e8befb8bff8345275cb03c3209b338f68b",
+    "zh:aa09ba1aa6fec278198ff352cc7f2977cfe567d31fd948c54fba5db82b4cd7ec",
+    "zh:ca28efbf60400918b9dadd18ecbf683065bf9329b35cbf3826718d8d50f10263",
+    "zh:cb21b119202ac6a30724beb89aefbb8660762b0e9b7165f1e22d59720dd0f110",
+    "zh:f36b4c9fe4795e892b3be2c80a22461f373541f81d335b51afa963097ab29624",
+  ]
+}
diff --git a/infra/minio/openobserve/openobserve.tf b/infra/minio/openobserve/openobserve.tf
new file mode 100644
index 00000000..a8944b4e
--- /dev/null
+++ b/infra/minio/openobserve/openobserve.tf
@@ -0,0 +1,64 @@
+resource "minio_s3_bucket" "openobserve" {
+  bucket = "openobserve"
+  acl    = "private"
+}
+
+resource "minio_iam_user" "openobserve" {
+  name          = "openobserve"
+  force_destroy = true
+  tags = {
+    env       = "prod"
+    managedBy = "terraform"
+  }
+}
+
+resource "minio_iam_policy" "openobserve" {
+  name   = "openobserve"
+  policy = <<EOF
+{
+  "Version":"2012-10-17",
+  "Statement": [
+    {
+      "Sid": "ObjectFullAccess",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:DeleteObject"
+      ],
+      "Effect": "Allow",
+      "Resource": "arn:aws:s3:::openobserve/*"
+    }
+  ]
+}
+EOF
+}
+
+resource "minio_iam_user_policy_attachment" "openobserve-1" {
+  user_name   = minio_iam_user.openobserve.id
+  policy_name = minio_iam_policy.openobserve.id
+}
+
+resource "minio_iam_service_account" "openobserve" {
+  target_user = minio_iam_user.openobserve.name
+}
+
+
+# ======================================================
+
+output "openobserve_id" {
+  value = minio_s3_bucket.openobserve.id
+}
+
+output "openobserve_url" {
+  value = minio_s3_bucket.openobserve.bucket_domain_name
+}
+
+output "openobserve_accesskey" {
+  value = minio_iam_service_account.openobserve.access_key
+}
+
+output "openobserve_secretkey" {
+  value     = minio_iam_service_account.openobserve.secret_key
+  sensitive = true
+}
diff --git a/infra/minio/openobserve/provider.tf b/infra/minio/openobserve/provider.tf
new file mode 100644
index 00000000..90a6b155
--- /dev/null
+++ b/infra/minio/openobserve/provider.tf
@@ -0,0 +1,41 @@
+terraform {
+  # https://developer.hashicorp.com/terraform/language/settings/backends/s3#credentials-and-shared-configuration
+  backend "s3" {
+    bucket = "tf-s3-backend"
+    key    = "homelab/minio/terraform.tfstate"
+    region = "us-east-1"
+    endpoints = {
+      s3 = "https://minio.writefor.fun"
+    }
+
+    # pass access key & secret via:
+    # 1. env: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+    # 2. aws credential: ~/.aws/credentials
+    # access_key = ""
+    # secret_key = ""
+
+    # we're using minio, skip all aws related validation & checks
+    skip_credentials_validation = true
+    skip_metadata_api_check     = true
+    skip_region_validation      = true
+    skip_requesting_account_id  = true
+    use_path_style              = true
+  }
+
+  required_providers {
+    minio = {
+      source  = "aminueza/minio"
+      version = "2.5.0"
+    }
+  }
+}
+
+# https://registry.terraform.io/providers/aminueza/minio/latest/docs
+provider "minio" {
+  minio_server = "minio.writefor.fun"
+  minio_user   = "ryan"
+
+  minio_api_version = "v4"
+  minio_region      = "us-east-1"
+  minio_ssl         = true
+}
diff --git a/infra/minio/openobserve/run.sh b/infra/minio/openobserve/run.sh
new file mode 100644
index 00000000..ee698601
--- /dev/null
+++ b/infra/minio/openobserve/run.sh
@@ -0,0 +1,12 @@
+# for provider
+#
+# export MINIO_PASSWORD=="xxx"
+
+# for terraform's s3 backend
+#
+# export AWS_ACCESS_KEY_ID="xxx"
+# export AWS_SECRET_ACCESS_KEY="xxx"
+#
+terraform init
+terraform plan
+terraform apply
diff --git a/infra/minio/tf-s3-backend/.terraform.lock.hcl b/infra/minio/tf-s3-backend/.terraform.lock.hcl
new file mode 100644
index 00000000..2579c2d3
--- /dev/null
+++ b/infra/minio/tf-s3-backend/.terraform.lock.hcl
@@ -0,0 +1,22 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/aminueza/minio" {
+  version     = "2.5.0"
+  constraints = "2.5.0"
+  hashes = [
+    "h1:RrjfsRy+fBVh7VF3r9u7uCCSjAdR5APa6sqbc9b8GfU=",
+    "zh:066cdb289dbfd1675e22fe58c8b42e2732f24fc1528b1919a78dfe28f80e8b30",
+    "zh:26d5e55106259e69493b95058178ec3d6b2395f03a8fe832af1be0e4d89ef42c",
+    "zh:6247e19de9ec6ef719cfcb174b8f08085c0fd5118b3b0de3fb9bb150702b4ad8",
+    "zh:70c3cbab0ba8edeec0db2e175bcdb47255c92f3153f839c4e8f2b0fe8c1366f4",
+    "zh:713793b4b93ae62070b18983ff525390de6c84547cab4220aa068437149f5035",
+    "zh:72de3e532d4bc7c7a4a872aaf00d7e4dfa09f3730668a738bb881d6734248f02",
+    "zh:9090f9288d7bc9f23043c1e65d8535e91f10413a16699d4a18add811b25fa167",
+    "zh:9847284aecb52718468feccb914d67e8befb8bff8345275cb03c3209b338f68b",
+    "zh:aa09ba1aa6fec278198ff352cc7f2977cfe567d31fd948c54fba5db82b4cd7ec",
+    "zh:ca28efbf60400918b9dadd18ecbf683065bf9329b35cbf3826718d8d50f10263",
+    "zh:cb21b119202ac6a30724beb89aefbb8660762b0e9b7165f1e22d59720dd0f110",
+    "zh:f36b4c9fe4795e892b3be2c80a22461f373541f81d335b51afa963097ab29624",
+  ]
+}
diff --git a/infra/minio/tf-s3-backend/README.md b/infra/minio/tf-s3-backend/README.md
new file mode 100644
index 00000000..1503c556
--- /dev/null
+++ b/infra/minio/tf-s3-backend/README.md
@@ -0,0 +1,5 @@
+# Terraform's S3 Backend
+
+This terraform workspace will be used only once, and we will not save the terrform.tfstate file.
+
+It's used to create a minio bucket to store all other tfstate files.
diff --git a/infra/minio/tf-s3-backend/provider.tf b/infra/minio/tf-s3-backend/provider.tf
new file mode 100644
index 00000000..f988f4b3
--- /dev/null
+++ b/infra/minio/tf-s3-backend/provider.tf
@@ -0,0 +1,18 @@
+terraform {
+  required_providers {
+    minio = {
+      source  = "aminueza/minio"
+      version = "2.5.0"
+    }
+  }
+}
+
+# https://registry.terraform.io/providers/aminueza/minio/latest/docs
+provider "minio" {
+  minio_server = "minio.writefor.fun"
+  minio_user   = "ryan"
+
+  minio_api_version = "v4"
+  minio_region      = "us-east-1"
+  minio_ssl         = true
+}
diff --git a/infra/minio/tf-s3-backend/run.sh b/infra/minio/tf-s3-backend/run.sh
new file mode 100644
index 00000000..4b10ff64
--- /dev/null
+++ b/infra/minio/tf-s3-backend/run.sh
@@ -0,0 +1,7 @@
+# for provider
+#
+# export MINIO_PASSWORD=="xxx"
+#
+terraform init
+terraform plan
+terraform apply
diff --git a/infra/minio/tf-s3-backend/tf-s3-backend.tf b/infra/minio/tf-s3-backend/tf-s3-backend.tf
new file mode 100644
index 00000000..bec970c3
--- /dev/null
+++ b/infra/minio/tf-s3-backend/tf-s3-backend.tf
@@ -0,0 +1,64 @@
+# https://developer.hashicorp.com/terraform/language/settings/backends/s3
+resource "minio_s3_bucket" "tf-s3-backend" {
+  bucket = "tf-s3-backend"
+  acl    = "private"
+}
+
+resource "minio_iam_user" "tf-s3-backend" {
+  name          = "tf-s3-backend"
+  force_destroy = true
+  tags = {
+    env       = "prod"
+    managedBy = "terraform"
+  }
+}
+
+resource "minio_iam_policy" "tf-s3-backend" {
+  name   = "tf-s3-backend"
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "s3:ListBucket",
+      "Resource": "arn:aws:s3:::tf-s3-backend"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
+      "Resource": "arn:aws:s3:::tf-s3-backend/*"
+    }
+  ]
+}
+EOF
+}
+
+resource "minio_iam_user_policy_attachment" "tf-s3-backend-1" {
+  user_name   = minio_iam_user.tf-s3-backend.id
+  policy_name = minio_iam_policy.tf-s3-backend.id
+}
+
+resource "minio_iam_service_account" "tf-s3-backend" {
+  target_user = minio_iam_user.tf-s3-backend.name
+}
+
+
+# ======================================================
+
+output "tf-s3-backend_id" {
+  value = minio_s3_bucket.tf-s3-backend.id
+}
+
+output "tf-s3-backend_url" {
+  value = minio_s3_bucket.tf-s3-backend.bucket_domain_name
+}
+
+output "tf-s3-backend_accesskey" {
+  value = minio_iam_service_account.tf-s3-backend.access_key
+}
+
+output "tf-s3-backend_secretkey" {
+  value     = minio_iam_service_account.tf-s3-backend.secret_key
+  sensitive = true
+}
diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix
index ffc7ed7f..30f6dea5 100644
--- a/lib/genK3sServerModule.nix
+++ b/lib/genK3sServerModule.nix
@@ -71,6 +71,7 @@ in {
 
   # create symlinks to link k3s's cni directory to the one used by almost all CNI plugins
   # such as multus, calico, etc.
+  # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
     "L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/current/bin"
     # If you have disabled flannel, you will have to create the directory via a tmpfiles rule