Merge pull request 'changed grafana persistence to postgresql db' (#29) from persist-grafana-data-on-postgres into main

02-k8s-bootstrap/CLAUDE.md

@@ -11,7 +11,8 @@ Bootstraps a k3s cluster on the Proxmox VMs created by `proxmox-infra`. Starts V
 3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
 4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
 5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes
-6. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
+6. Formats the workers' dedicated `scsi1` disk (ext4, if not already formatted) and mounts it at `/mnt/longhorn-extra` (via `/etc/fstab`) — this is the disk `03-k8s-infra` later registers as an extra Longhorn disk, to keep Longhorn off the root filesystem and avoid `DiskPressure`
+7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
 
 VM IDs and the CI runner SSH private key are read automatically from the `proxmox-infra` stack output via StackReference — no manual setup needed for those.
 

02-k8s-bootstrap/index.ts

@@ -215,7 +215,32 @@ const joinWorker2 = new command.remote.Command(
 );
 
 // ---------------------------------------------------------------------------
-// Step 5: Read kubeconfig from master-1, patch server URL for external access
+// Step 5: Format and mount Longhorn disks
+// ---------------------------------------------------------------------------
+
+const workers = [worker1Ip, worker2Ip];
+const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh];
+
+const mountLonghornDisks = workers.map((worker, i) => {
+  return new command.remote.Command(
+    `mount-longhorn-disk-${i}`,
+    {
+      connection: conn(worker),
+      create: `set -euo pipefail; 
+        DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1'); 
+        DISK_PATH="/dev/disk/by-id/$DISK_ID"; 
+        sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH"; 
+        sudo mkdir -p /mnt/longhorn-extra; 
+        echo ""$DISK_PATH" /mnt/longhorn-extra  ext4  defaults,nofail  0  2" | sudo tee -a /etc/fstab; 
+        sudo mount -a; 
+        df -h /mnt/longhorn-extra`,
+    },
+    { dependsOn: workerSshReader[i] },
+  );
+});
+
+// ---------------------------------------------------------------------------
+// Step 6: Read kubeconfig from master-1, patch server URL for external access
 // ---------------------------------------------------------------------------
 
 const getKubeconfig = new command.remote.Command(

03-k8s-infra/CLAUDE.md

@@ -21,7 +21,7 @@ CI installs the `helm` CLI (`azure/setup-helm`) before running Pulumi.
 | `truenas-nfs` StorageClass | Backed by an NFS share on TrueNAS (`Retain` reclaim policy) |
 | cert-manager | Certificate lifecycle management |
 | `letsencrypt-prod` ClusterIssuer | DNS-01 via Cloudflare, issues Let's Encrypt TLS certs |
-| Longhorn | Default block storage (`longhorn` StorageClass, 3 replicas) — used by app/monitoring PVCs |
+| Longhorn | Default block storage (`longhorn` StorageClass, 2 replicas) — used by app/monitoring PVCs. Worker nodes get an extra `longhorn-extra` disk at `/mnt/longhorn-extra` (a dedicated mount set up by `k8s-bootstrap`, patched onto the Longhorn `Node` CR here) so Longhorn data stays off the root disk and doesn't cause `DiskPressure`. `nodeDownPodDeletionPolicy` deletes both the StatefulSet and Deployment pod on node-down so stateful workloads reschedule automatically |
 | kube-vip | DaemonSet that ARP-announces a virtual IP for LoadBalancer Services |
 | kube-vip-cloud-provider | Allocates LoadBalancer IPs from a single-address pool (`kubeVipAddress`) — k3s's built-in ServiceLB is not used |
 | `traefik` ServicePatch | Pins Traefik's LoadBalancer IP to `kubeVipAddress` |

03-k8s-infra/index.ts

@@ -122,7 +122,7 @@ new k8s.apiextensions.CustomResource(
 
 // ── Longhorn ──────────────────────────────────────────────────────────────
 
- new k8s.helm.v3.Release(
+const longhorn = new k8s.helm.v3.Release(
   "longhorn",
   {
     name: "longhorn",
@@ -133,14 +133,46 @@ new k8s.apiextensions.CustomResource(
     version: "1.12.0",
     values: {
       defaultSettings: {
-        defaultReplicaCount: 3,
+        defaultReplicaCount: 2,
-        podDeletionPolicyWhenNodeIsDown: "delete-pod-and-force-delete-pod",
+        nodeDownPodDeletionPolicy: "delete-both-statefulset-and-deployment-pod",
+        createDefaultDiskLabeledNodes: true,
       },
     },
   },
   opts(),
 );
 
+// Patches Longhorn instances to use the mountend Longhorn storage on the worker nodes.
+["k3s-worker-1", "k3s-worker-2"].forEach(
+  (nodeName) =>
+    new k8s.apiextensions.CustomResourcePatch(
+      `${nodeName}-longhorn-disks`,
+      {
+        apiVersion: "longhorn.io/v1beta2",
+        kind: "Node",
+        metadata: {
+          name: nodeName,
+          namespace: "longhorn-system",
+          annotations: { "pulumi.com/patchForce": "true" },
+        },
+        spec: {
+          disks: {
+            "longhorn-extra": {
+              allowScheduling: true,
+              diskDriver: "",
+              diskType: "filesystem",
+              evictionRequested: false,
+              path: "/mnt/longhorn-extra",
+              storageReserved: 0,
+              tags: [],
+            },
+          },
+        },
+      },
+      opts({ dependsOn: [longhorn] }),
+    ),
+);
+
 // ── kube-vip ────────────
 
 const kubeVipNamespace = "kube-system";
@@ -149,6 +181,22 @@ const controlPlaneTolerations = [
   { key: "node-role.kubernetes.io/control-plane", effect: "NoSchedule" },
 ];
 
+// Keeps regular workloads off the master nodes; kube-vip and
+// kube-vip-cloud-provider below explicitly tolerate these same taints.
+["k3s-master-1", "k3s-master-2", "k3s-master-3"].forEach((nodeName) => {
+  new k8s.core.v1.NodePatch(
+    `${nodeName}-control-plane-taint`,
+    {
+      metadata: {
+        name: nodeName,
+        annotations: { "pulumi.com/patchForce": "true" },
+      },
+      spec: { taints: controlPlaneTolerations },
+    },
+    opts(),
+  );
+});
+
 const kubeVipServiceAccount = new k8s.core.v1.ServiceAccount(
   "kube-vip",
   { metadata: { name: "kube-vip", namespace: kubeVipNamespace } },

05-monitoring/index.ts

@@ -13,6 +13,10 @@ const appsRef = new pulumi.StackReference(
 const kubeconfig = bootstrapRef.requireOutput("kubeconfig");
 const domain = appsRef.requireOutput("domain");
 const grafanaAdminPassword = config.requireSecret("grafanaAdminPassword");
+const grafanaDbUser = config.requireSecret("grafanaDbUser");
+const grafanaDbPassword = config.requireSecret("grafanaDbPassword");
+const grafanaDbHost = config.requireSecret("grafanaDbHost");
+const grafanaDbName = config.require("grafanaDbName");
 
 const k8sProvider = new k8s.Provider("k3s", { kubeconfig });
 
@@ -24,6 +28,15 @@ const monitoringNs = new k8s.core.v1.Namespace(
 
 const grafanaHost = pulumi.interpolate`grafana.${domain}`;
 
+const grafanaDbSecret = new k8s.core.v1.Secret(
+  "grafana-db-credentials",
+  {
+    metadata: { namespace: monitoringNs.metadata.name },
+    stringData: { password: grafanaDbPassword },
+  },
+  { provider: k8sProvider },
+);
+
 new k8s.helm.v3.Release(
   "kube-prometheus-stack",
   {
@@ -38,6 +51,7 @@ new k8s.helm.v3.Release(
     values: {
       grafana: {
         adminPassword: grafanaAdminPassword,
+        replicas: 2,
         ingress: {
           enabled: true,
           ingressClassName: "traefik",
@@ -45,14 +59,29 @@ new k8s.helm.v3.Release(
           hosts: [grafanaHost],
           tls: [{ secretName: "grafana-tls", hosts: [grafanaHost] }],
         },
-        persistence: {
+        env: {
-          enabled: true,
+          GF_DATABASE_TYPE: "postgres",
-          storageClassName: "longhorn",
+          GF_DATABASE_HOST: grafanaDbHost,
-          size: "2Gi",
+          GF_DATABASE_NAME: grafanaDbName,
+          GF_DATABASE_USER: grafanaDbUser,
+          GF_DATABASE_SSL_MODE: "disable",
+        },
+        envValueFrom: {
+          GF_DATABASE_PASSWORD: {
+            secretKeyRef: {
+              name: grafanaDbSecret.metadata.name,
+              key: "password",
+            },
+          },
         },
       },
       prometheus: {
         prometheusSpec: {
+          retention: "7d",
+          retentionSize: "15GB",
+          scrapeInterval: "60s",
+          evaluationInterval: "60s",
+          replicas: "2",
           storageSpec: {
             volumeClaimTemplate: {
               spec: {