Merge pull request 'changed grafana persistence to postgresql db' (#29) from persist-grafana-data-on-postgres into main

02-k8s-bootstrap/CLAUDE.md

@@ -11,7 +11,8 @@ Bootstraps a k3s cluster on the Proxmox VMs created by `proxmox-infra`. Starts V
 3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
 4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
 5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes
-6. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
+6. Formats the workers' dedicated `scsi1` disk (ext4, if not already formatted) and mounts it at `/mnt/longhorn-extra` (via `/etc/fstab`) — this is the disk `03-k8s-infra` later registers as an extra Longhorn disk, to keep Longhorn off the root filesystem and avoid `DiskPressure`
+7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
 
 VM IDs and the CI runner SSH private key are read automatically from the `proxmox-infra` stack output via StackReference — no manual setup needed for those.
 

03-k8s-infra/CLAUDE.md

@@ -21,7 +21,7 @@ CI installs the `helm` CLI (`azure/setup-helm`) before running Pulumi.
 | `truenas-nfs` StorageClass | Backed by an NFS share on TrueNAS (`Retain` reclaim policy) |
 | cert-manager | Certificate lifecycle management |
 | `letsencrypt-prod` ClusterIssuer | DNS-01 via Cloudflare, issues Let's Encrypt TLS certs |
-| Longhorn | Default block storage (`longhorn` StorageClass, 3 replicas) — used by app/monitoring PVCs |
+| Longhorn | Default block storage (`longhorn` StorageClass, 2 replicas) — used by app/monitoring PVCs. Worker nodes get an extra `longhorn-extra` disk at `/mnt/longhorn-extra` (a dedicated mount set up by `k8s-bootstrap`, patched onto the Longhorn `Node` CR here) so Longhorn data stays off the root disk and doesn't cause `DiskPressure`. `nodeDownPodDeletionPolicy` deletes both the StatefulSet and Deployment pod on node-down so stateful workloads reschedule automatically |
 | kube-vip | DaemonSet that ARP-announces a virtual IP for LoadBalancer Services |
 | kube-vip-cloud-provider | Allocates LoadBalancer IPs from a single-address pool (`kubeVipAddress`) — k3s's built-in ServiceLB is not used |
 | `traefik` ServicePatch | Pins Traefik's LoadBalancer IP to `kubeVipAddress` |

03-k8s-infra/index.ts

@@ -143,7 +143,7 @@ const longhorn = new k8s.helm.v3.Release(
 );
 
 // Patches Longhorn instances to use the mountend Longhorn storage on the worker nodes.
-const workerLonghornDiskPatches = ["k3s-worker-1", "k3s-worker-2"].map(
+["k3s-worker-1", "k3s-worker-2"].forEach(
   (nodeName) =>
     new k8s.apiextensions.CustomResourcePatch(
       `${nodeName}-longhorn-disks`,
@@ -181,6 +181,22 @@ const controlPlaneTolerations = [
   { key: "node-role.kubernetes.io/control-plane", effect: "NoSchedule" },
 ];
 
+// Keeps regular workloads off the master nodes; kube-vip and
+// kube-vip-cloud-provider below explicitly tolerate these same taints.
+["k3s-master-1", "k3s-master-2", "k3s-master-3"].forEach((nodeName) => {
+  new k8s.core.v1.NodePatch(
+    `${nodeName}-control-plane-taint`,
+    {
+      metadata: {
+        name: nodeName,
+        annotations: { "pulumi.com/patchForce": "true" },
+      },
+      spec: { taints: controlPlaneTolerations },
+    },
+    opts(),
+  );
+});
+
 const kubeVipServiceAccount = new k8s.core.v1.ServiceAccount(
   "kube-vip",
   { metadata: { name: "kube-vip", namespace: kubeVipNamespace } },

05-monitoring/index.ts

@@ -13,6 +13,10 @@ const appsRef = new pulumi.StackReference(
 const kubeconfig = bootstrapRef.requireOutput("kubeconfig");
 const domain = appsRef.requireOutput("domain");
 const grafanaAdminPassword = config.requireSecret("grafanaAdminPassword");
+const grafanaDbUser = config.requireSecret("grafanaDbUser");
+const grafanaDbPassword = config.requireSecret("grafanaDbPassword");
+const grafanaDbHost = config.requireSecret("grafanaDbHost");
+const grafanaDbName = config.require("grafanaDbName");
 
 const k8sProvider = new k8s.Provider("k3s", { kubeconfig });
 
@@ -24,6 +28,15 @@ const monitoringNs = new k8s.core.v1.Namespace(
 
 const grafanaHost = pulumi.interpolate`grafana.${domain}`;
 
+const grafanaDbSecret = new k8s.core.v1.Secret(
+  "grafana-db-credentials",
+  {
+    metadata: { namespace: monitoringNs.metadata.name },
+    stringData: { password: grafanaDbPassword },
+  },
+  { provider: k8sProvider },
+);
+
 new k8s.helm.v3.Release(
   "kube-prometheus-stack",
   {
@@ -38,6 +51,7 @@ new k8s.helm.v3.Release(
     values: {
       grafana: {
         adminPassword: grafanaAdminPassword,
+        replicas: 2,
         ingress: {
           enabled: true,
           ingressClassName: "traefik",
@@ -45,14 +59,29 @@ new k8s.helm.v3.Release(
           hosts: [grafanaHost],
           tls: [{ secretName: "grafana-tls", hosts: [grafanaHost] }],
         },
-        persistence: {
-          enabled: true,
-          storageClassName: "longhorn",
-          size: "2Gi",
+        env: {
+          GF_DATABASE_TYPE: "postgres",
+          GF_DATABASE_HOST: grafanaDbHost,
+          GF_DATABASE_NAME: grafanaDbName,
+          GF_DATABASE_USER: grafanaDbUser,
+          GF_DATABASE_SSL_MODE: "disable",
+        },
+        envValueFrom: {
+          GF_DATABASE_PASSWORD: {
+            secretKeyRef: {
+              name: grafanaDbSecret.metadata.name,
+              key: "password",
+            },
+          },
         },
       },
       prometheus: {
         prometheusSpec: {
+          retention: "7d",
+          retentionSize: "15GB",
+          scrapeInterval: "60s",
+          evaluationInterval: "60s",
+          replicas: "2",
           storageSpec: {
             volumeClaimTemplate: {
               spec: {