Merge pull request 'changed grafana persistence to postgresql db' (#29) from persist-grafana-data-on-postgres into main

Reviewed-on: #26

01-proxmox-infra/index.ts

@@ -303,7 +303,7 @@ k3sVms.forEach((vmResource, i) => {
     (nic) => nic[0].macAddress,
   );
 
-  return new pfsense.Dhcpv4Staticmapping(
+  new pfsense.Dhcpv4Staticmapping(
     `${nodeConfigs[i].name}-dhcp`,
     {
       interface: "lan",
@@ -315,8 +315,6 @@ k3sVms.forEach((vmResource, i) => {
   );
 });
 
-// Individual vmId exports — used by k8s-bootstrap to start VMs.
-// Order matches nodeConfigs: master-1, master-2, worker-1, master-3, worker-2.
 export const vmIds = {
   master1: k3sVms[0].vmId,
   master2: k3sVms[1].vmId,
@@ -328,8 +326,6 @@ export const vmIds = {
 // CI runner SSH private key — consumed by k8s-bootstrap via StackReference.
 export const ciRunnerPrivateKey = pulumi.secret(ciRunnerKey.privateKeyOpenssh);
 
-// Proxmox API credentials — consumed by k8s-bootstrap via StackReference.
 export { pve1Endpoint, pve1ApiToken, pve2Endpoint, pve2ApiToken };
 
-//k3s instance ips consumed by k8s-bootstrap.
 export { master1Ip, master2Ip, worker1Ip, master3Ip, worker2Ip };

02-k8s-bootstrap/CLAUDE.md

@@ -11,7 +11,8 @@ Bootstraps a k3s cluster on the Proxmox VMs created by `proxmox-infra`. Starts V
 3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
 4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
 5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes
-6. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
+6. Formats the workers' dedicated `scsi1` disk (ext4, if not already formatted) and mounts it at `/mnt/longhorn-extra` (via `/etc/fstab`) — this is the disk `03-k8s-infra` later registers as an extra Longhorn disk, to keep Longhorn off the root filesystem and avoid `DiskPressure`
+7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
 
 VM IDs and the CI runner SSH private key are read automatically from the `proxmox-infra` stack output via StackReference — no manual setup needed for those.
 

02-k8s-bootstrap/index.ts

@@ -215,7 +215,32 @@ const joinWorker2 = new command.remote.Command(
 );
 
 // ---------------------------------------------------------------------------
-// Step 5: Read kubeconfig from master-1, patch server URL for external access
+// Step 5: Format and mount Longhorn disks
+// ---------------------------------------------------------------------------
+
+const workers = [worker1Ip, worker2Ip];
+const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh];
+
+const mountLonghornDisks = workers.map((worker, i) => {
+  return new command.remote.Command(
+    `mount-longhorn-disk-${i}`,
+    {
+      connection: conn(worker),
+      create: `set -euo pipefail; 
+        DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1'); 
+        DISK_PATH="/dev/disk/by-id/$DISK_ID"; 
+        sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH"; 
+        sudo mkdir -p /mnt/longhorn-extra; 
+        echo ""$DISK_PATH" /mnt/longhorn-extra  ext4  defaults,nofail  0  2" | sudo tee -a /etc/fstab; 
+        sudo mount -a; 
+        df -h /mnt/longhorn-extra`,
+    },
+    { dependsOn: workerSshReader[i] },
+  );
+});
+
+// ---------------------------------------------------------------------------
+// Step 6: Read kubeconfig from master-1, patch server URL for external access
 // ---------------------------------------------------------------------------
 
 const getKubeconfig = new command.remote.Command(

03-k8s-infra/CLAUDE.md

@@ -21,12 +21,12 @@ CI installs the `helm` CLI (`azure/setup-helm`) before running Pulumi.
 | `truenas-nfs` StorageClass | Backed by an NFS share on TrueNAS (`Retain` reclaim policy) |
 | cert-manager | Certificate lifecycle management |
 | `letsencrypt-prod` ClusterIssuer | DNS-01 via Cloudflare, issues Let's Encrypt TLS certs |
-| Longhorn | Default block storage (`longhorn` StorageClass, 3 replicas) — used by app/monitoring PVCs |
+| Longhorn | Default block storage (`longhorn` StorageClass, 2 replicas) — used by app/monitoring PVCs. Worker nodes get an extra `longhorn-extra` disk at `/mnt/longhorn-extra` (a dedicated mount set up by `k8s-bootstrap`, patched onto the Longhorn `Node` CR here) so Longhorn data stays off the root disk and doesn't cause `DiskPressure`. `nodeDownPodDeletionPolicy` deletes both the StatefulSet and Deployment pod on node-down so stateful workloads reschedule automatically |
 | kube-vip | DaemonSet that ARP-announces a virtual IP for LoadBalancer Services |
 | kube-vip-cloud-provider | Allocates LoadBalancer IPs from a single-address pool (`kubeVipAddress`) — k3s's built-in ServiceLB is not used |
 | `traefik` ServicePatch | Pins Traefik's LoadBalancer IP to `kubeVipAddress` |
 
-**Export**: `storageClass = "truenas-nfs"`.
+**Export**: `nfsStorageClass = "truenas-nfs"`.
 
 ## Required Pulumi config
 

03-k8s-infra/index.ts

@@ -8,10 +8,14 @@ const bootstrapRef = new pulumi.StackReference(
 );
 
 const kubeconfig = bootstrapRef.requireOutput("kubeconfig");
+
+export const nfsStorageClass = "truenas-nfs";
 const truenasHost = config.requireSecret("truenasHost");
 const truenasNfsPath = config.requireSecret("truenasNfsPath");
+
 const cloudflareToken = config.requireSecret("cloudflareApiToken");
 const letsencryptEmail = config.requireSecret("letsencryptEmail");
+
 const kubeVipAddress = config.requireSecret("kubeVipAddress");
 const kubeVipInterface = config.require("kubeVipInterface");
 
@@ -41,9 +45,9 @@ const nfsCsiDriver = new k8s.helm.v3.Release(
 );
 
 new k8s.storage.v1.StorageClass(
-  "truenas-nfs",
+  nfsStorageClass,
   {
-    metadata: { name: "truenas-nfs" },
+    metadata: { name: nfsStorageClass },
     provisioner: "nfs.csi.k8s.io",
     parameters: {
       server: truenasHost,
@@ -118,7 +122,7 @@ new k8s.apiextensions.CustomResource(
 
 // ── Longhorn ──────────────────────────────────────────────────────────────
 
-new k8s.helm.v3.Release(
+const longhorn = new k8s.helm.v3.Release(
   "longhorn",
   {
     name: "longhorn",
@@ -129,13 +133,46 @@ new k8s.helm.v3.Release(
     version: "1.12.0",
     values: {
       defaultSettings: {
-        defaultReplicaCount: 3,
+        defaultReplicaCount: 2,
+        nodeDownPodDeletionPolicy: "delete-both-statefulset-and-deployment-pod",
+        createDefaultDiskLabeledNodes: true,
       },
     },
   },
   opts(),
 );
 
+// Patches Longhorn instances to use the mountend Longhorn storage on the worker nodes.
+["k3s-worker-1", "k3s-worker-2"].forEach(
+  (nodeName) =>
+    new k8s.apiextensions.CustomResourcePatch(
+      `${nodeName}-longhorn-disks`,
+      {
+        apiVersion: "longhorn.io/v1beta2",
+        kind: "Node",
+        metadata: {
+          name: nodeName,
+          namespace: "longhorn-system",
+          annotations: { "pulumi.com/patchForce": "true" },
+        },
+        spec: {
+          disks: {
+            "longhorn-extra": {
+              allowScheduling: true,
+              diskDriver: "",
+              diskType: "filesystem",
+              evictionRequested: false,
+              path: "/mnt/longhorn-extra",
+              storageReserved: 0,
+              tags: [],
+            },
+          },
+        },
+      },
+      opts({ dependsOn: [longhorn] }),
+    ),
+);
+
 // ── kube-vip ────────────
 
 const kubeVipNamespace = "kube-system";
@@ -144,6 +181,22 @@ const controlPlaneTolerations = [
   { key: "node-role.kubernetes.io/control-plane", effect: "NoSchedule" },
 ];
 
+// Keeps regular workloads off the master nodes; kube-vip and
+// kube-vip-cloud-provider below explicitly tolerate these same taints.
+["k3s-master-1", "k3s-master-2", "k3s-master-3"].forEach((nodeName) => {
+  new k8s.core.v1.NodePatch(
+    `${nodeName}-control-plane-taint`,
+    {
+      metadata: {
+        name: nodeName,
+        annotations: { "pulumi.com/patchForce": "true" },
+      },
+      spec: { taints: controlPlaneTolerations },
+    },
+    opts(),
+  );
+});
+
 const kubeVipServiceAccount = new k8s.core.v1.ServiceAccount(
   "kube-vip",
   { metadata: { name: "kube-vip", namespace: kubeVipNamespace } },
@@ -321,13 +374,20 @@ const cloudProviderClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding(
   opts(),
 );
 
+const kubeVipCloudProviderLabels = {
+  app: "kube-vip",
+  component: "kube-vip-cloud-provider",
+};
+
 // IP pool the cloud-provider allocates from — scoped to exactly the one VIP we
 // want, so it can never hand the address to some other LoadBalancer Service.
 const kubeVipPool = new k8s.core.v1.ConfigMap(
   "kube-vip-pool",
   {
     metadata: { name: "kubevip", namespace: kubeVipNamespace },
-    data: { "range-global": pulumi.interpolate`${kubeVipAddress}-${kubeVipAddress}` },
+    data: {
+      "range-global": pulumi.interpolate`${kubeVipAddress}-${kubeVipAddress}`,
+    },
   },
   opts(),
 );
@@ -339,11 +399,11 @@ const kubeVipCloudProvider = new k8s.apps.v1.Deployment(
     spec: {
       replicas: 1,
       selector: {
-        matchLabels: { app: "kube-vip", component: "kube-vip-cloud-provider" },
+        matchLabels: kubeVipCloudProviderLabels,
       },
       template: {
         metadata: {
-          labels: { app: "kube-vip", component: "kube-vip-cloud-provider" },
+          labels: kubeVipCloudProviderLabels,
         },
         spec: {
           serviceAccountName: cloudProviderServiceAccount.metadata.name,
@@ -378,5 +438,3 @@ new k8s.core.v1.ServicePatch(
   },
   opts({ dependsOn: [kubeVip, kubeVipCloudProvider] }),
 );
-
-export const storageClass = "truenas-nfs";

04-k8s-apps/index.ts

@@ -16,7 +16,7 @@ const opts = (extras?: pulumi.ResourceOptions): pulumi.ResourceOptions => ({
   ...extras,
 });
 
-const ns = new k8s.core.v1.Namespace(
+const heimdallNs = new k8s.core.v1.Namespace(
   "heimdall",
   { metadata: { name: "heimdall" } },
   opts(),
@@ -25,7 +25,7 @@ const ns = new k8s.core.v1.Namespace(
 const pvc = new k8s.core.v1.PersistentVolumeClaim(
   "heimdall-config",
   {
-    metadata: { name: "heimdall-config", namespace: ns.metadata.name },
+    metadata: { name: "heimdall-config", namespace: heimdallNs.metadata.name },
     spec: {
       accessModes: ["ReadWriteOnce"],
       storageClassName: "longhorn",
@@ -42,7 +42,7 @@ new k8s.apps.v1.Deployment(
   {
     metadata: {
       name: "heimdall",
-      namespace: ns.metadata.name,
+      namespace: heimdallNs.metadata.name,
       annotations: { "pulumi.com/patchForce": "true" },
     },
     spec: {
@@ -80,7 +80,7 @@ new k8s.apps.v1.Deployment(
 const svc = new k8s.core.v1.Service(
   "heimdall",
   {
-    metadata: { name: "heimdall", namespace: ns.metadata.name },
+    metadata: { name: "heimdall", namespace: heimdallNs.metadata.name },
     spec: {
       selector: labels,
       ports: [{ port: 80, targetPort: 80 }],
@@ -97,7 +97,7 @@ new k8s.networking.v1.Ingress(
   {
     metadata: {
       name: "heimdall",
-      namespace: ns.metadata.name,
+      namespace: heimdallNs.metadata.name,
       annotations: {
         "cert-manager.io/cluster-issuer": "letsencrypt-prod",
       },

05-monitoring/index.ts

@@ -13,10 +13,14 @@ const appsRef = new pulumi.StackReference(
 const kubeconfig = bootstrapRef.requireOutput("kubeconfig");
 const domain = appsRef.requireOutput("domain");
 const grafanaAdminPassword = config.requireSecret("grafanaAdminPassword");
+const grafanaDbUser = config.requireSecret("grafanaDbUser");
+const grafanaDbPassword = config.requireSecret("grafanaDbPassword");
+const grafanaDbHost = config.requireSecret("grafanaDbHost");
+const grafanaDbName = config.require("grafanaDbName");
 
 const k8sProvider = new k8s.Provider("k3s", { kubeconfig });
 
-const ns = new k8s.core.v1.Namespace(
+const monitoringNs = new k8s.core.v1.Namespace(
   "monitoring",
   { metadata: { name: "monitoring" } },
   { provider: k8sProvider, import: "monitoring" },
@@ -24,13 +28,22 @@ const ns = new k8s.core.v1.Namespace(
 
 const grafanaHost = pulumi.interpolate`grafana.${domain}`;
 
+const grafanaDbSecret = new k8s.core.v1.Secret(
+  "grafana-db-credentials",
+  {
+    metadata: { namespace: monitoringNs.metadata.name },
+    stringData: { password: grafanaDbPassword },
+  },
+  { provider: k8sProvider },
+);
+
 new k8s.helm.v3.Release(
   "kube-prometheus-stack",
   {
     name: "kube-prometheus-stack",
     chart: "kube-prometheus-stack",
     version: "86.2.0",
-    namespace: ns.metadata.name,
+    namespace: monitoringNs.metadata.name,
     repositoryOpts: {
       repo: "https://prometheus-community.github.io/helm-charts",
     },
@@ -38,6 +51,7 @@ new k8s.helm.v3.Release(
     values: {
       grafana: {
         adminPassword: grafanaAdminPassword,
+        replicas: 2,
         ingress: {
           enabled: true,
           ingressClassName: "traefik",
@@ -45,14 +59,29 @@ new k8s.helm.v3.Release(
           hosts: [grafanaHost],
           tls: [{ secretName: "grafana-tls", hosts: [grafanaHost] }],
         },
-        persistence: {
-          enabled: true,
-          storageClassName: "longhorn",
-          size: "2Gi",
+        env: {
+          GF_DATABASE_TYPE: "postgres",
+          GF_DATABASE_HOST: grafanaDbHost,
+          GF_DATABASE_NAME: grafanaDbName,
+          GF_DATABASE_USER: grafanaDbUser,
+          GF_DATABASE_SSL_MODE: "disable",
+        },
+        envValueFrom: {
+          GF_DATABASE_PASSWORD: {
+            secretKeyRef: {
+              name: grafanaDbSecret.metadata.name,
+              key: "password",
+            },
+          },
         },
       },
       prometheus: {
         prometheusSpec: {
+          retention: "7d",
+          retentionSize: "15GB",
+          scrapeInterval: "60s",
+          evaluationInterval: "60s",
+          replicas: "2",
           storageSpec: {
             volumeClaimTemplate: {
               spec: {