docs(k8s-bootstrap): document control-plane VIP and pfSense HAProxy setup

The VIP and HAProxy frontend/backend aren't exposed by the pfsense provider, so they must be created by hand before this stack is deployed.
feature(k8s-bootstrap): join nodes and export kubeconfig via control-plane VIP
2026-10-02 23:52:32 +02:00 · 2026-10-02 23:52:32 +02:00 · 2026-08-05 16:39:09 +02:00 · 2026-08-05 16:32:07 +02:00 · 2026-08-05 16:31:38 +02:00 · 2026-07-30 02:55:01 +02:00
5 changed files with 363 additions and 228 deletions
@@ -25,33 +25,121 @@ const worker1Ip = config.requireSecret("worker1Ip");
 const worker2Ip = config.requireSecret("worker2Ip");
 const worker3Ip = config.requireSecret("worker3Ip");

+// ---------------------------------------------------------------------------
+// VM templates — one per node, cloned from the downloaded cloud image.
+// Templates are not started and have no cloud-init config; that is applied
+// per-clone so each node gets its own hostname (derived from VM name).
+// ---------------------------------------------------------------------------
+
+const templateSettings = {
+  template: true,
+  started: false,
+  stopOnDestroy: true,
+  scsiHardware: "virtio-scsi-pci",
+  cdrom: { fileId: "none" },
+  cpu: {
+    cores: 2,
+    sockets: 1,
+    type: "host",
+    numa: true,
+  },
+  memory: {
+    dedicated: 2048,
+    floating: 0,
+  },
+  networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
+  serialDevices: [{}],
+  vga: { type: "serial0" },
+  agent: { enabled: true, timeout: "30s" },
+};
+
 // ---------------------------------------------------------------------------
 // Providers — one per standalone Proxmox machine
 // ---------------------------------------------------------------------------

-const pveProvider = new proxmox.Provider("pve", {
-  endpoint: pve1Endpoint,
-  apiToken: pve1ApiToken,
-  insecure: true,
-});
+interface PveNodeConfig {
+  nodeName: string;
+  hostLabel: string;
+  isoStore: string;
+  dataStore: string;
+  endpoint: pulumi.Output<string>;
+  apiToken: pulumi.Output<string>;
+}

-const pveBckpProvider = new proxmox.Provider("pve-bckp", {
-  endpoint: pve2Endpoint,
-  apiToken: pve2ApiToken,
-  insecure: true,
-});
+const pveNodeSeeds: PveNodeConfig[] = [
+  {
+    nodeName: "pve",
+    hostLabel: "pve-b550",
+    isoStore: "pve-local-ext1",
+    dataStore: "local-lvm",
+    endpoint: pve1Endpoint,
+    apiToken: pve1ApiToken,
+  },
+  {
+    nodeName: "pve-bckp",
+    hostLabel: "pve-optiplex",
+    isoStore: "local",
+    dataStore: "local",
+    endpoint: pve2Endpoint,
+    apiToken: pve2ApiToken,
+  },
+  {
+    nodeName: "pve",
+    hostLabel: "pve-esprimo",
+    isoStore: "local",
+    dataStore: "local-lvm",
+    endpoint: pve3Endpoint,
+    apiToken: pve3ApiToken,
+  },
+];

-const pveEsprimoProvider = new proxmox.Provider("pve-esprimo", {
-  endpoint: pve3Endpoint,
-  apiToken: pve3ApiToken,
-  insecure: true,
-});
-
-const pveNodes = {
-  "pve-b550": { nodeName: "pve", isoStore: "pve-local-ext1", dataStore: "local-lvm" },
-  "pve-optiplex": { nodeName: "pve-bckp", isoStore: "local", dataStore: "local" },
-  "pve-esprimo": { nodeName: "pve", isoStore: "local", dataStore: "local-lvm" },
-} as const;
+const pveNodes = pveNodeSeeds
+  .map((n) => ({
+    ...n,
+    provider: new proxmox.Provider(n.hostLabel, {
+      endpoint: n.endpoint,
+      apiToken: n.apiToken,
+      insecure: true,
+    }),
+  }))
+  .map((n) => ({
+    ...n,
+    ubuntuImagePve: new proxmox.download.File(
+      `ubuntu-noble-${n.hostLabel}`,
+      {
+        nodeName: n.nodeName,
+        datastoreId: n.isoStore,
+        contentType: "import",
+        fileName: "noble-server-cloudimg-amd64.qcow2",
+        url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img",
+        overwrite: false,
+        overwriteUnmanaged: true,
+      },
+      { provider: n.provider },
+    ),
+  }))
+  .map((n) => ({
+    ...n,
+    pveTemplate: new proxmox.VmLegacy(
+      `k3s-template-${n.hostLabel}`,
+      {
+        ...templateSettings,
+        nodeName: n.nodeName,
+        name: "k3s-ubuntu-noble-template",
+        disks: [
+          {
+            interface: "scsi0",
+            datastoreId: n.dataStore,
+            importFrom: pulumi.interpolate`${n.ubuntuImagePve.datastoreId}:import/${n.ubuntuImagePve.fileName}`,
+            size: 10,
+            ssd: true,
+            discard: "on",
+          },
+        ],
+      },
+      { provider: n.provider, ignoreChanges: ["disks"] },
+    ),
+  }));

 // ---------------------------------------------------------------------------
 // Providers — PfSense
@@ -73,215 +161,62 @@ const ciRunnerKey = new tls.PrivateKey("ci-runner-key", {
  algorithm: "ED25519",
 });

-// ---------------------------------------------------------------------------
-// Download Ubuntu Noble cloud image to each node's ISO storage
-// ---------------------------------------------------------------------------
-
-const ubuntuNobleUrl =
-  "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img";
-
-const ubuntuImagePve = new proxmox.download.File(
-  "ubuntu-noble-pve",
-  {
-    nodeName: pveNodes["pve-b550"].nodeName,
-    datastoreId: pveNodes["pve-b550"].isoStore,
-    contentType: "import",
-    fileName: "noble-server-cloudimg-amd64.qcow2",
-    url: ubuntuNobleUrl,
-    overwrite: false,
-    overwriteUnmanaged: true,
-  },
-  { provider: pveProvider },
-);
-
-const ubuntuImagePveBckp = new proxmox.download.File(
-  "ubuntu-noble-pve-bckp",
-  {
-    nodeName: pveNodes["pve-optiplex"].nodeName,
-    datastoreId: pveNodes["pve-optiplex"].isoStore,
-    contentType: "import",
-    fileName: "noble-server-cloudimg-amd64.qcow2",
-    url: ubuntuNobleUrl,
-    overwrite: false,
-    overwriteUnmanaged: true,
-  },
-  { provider: pveBckpProvider },
-);
-
-const ubuntuImagePveEsprimo = new proxmox.download.File(
-  "ubuntu-noble-pve-esprimo",
-  {
-    nodeName: pveNodes["pve-esprimo"].nodeName,
-    datastoreId: pveNodes["pve-esprimo"].isoStore,
-    contentType: "import",
-    fileName: "noble-server-cloudimg-amd64.qcow2",
-    url: ubuntuNobleUrl,
-    overwrite: false,
-    overwriteUnmanaged: true,
-  },
-  { provider: pveEsprimoProvider },
-);
-
-// ---------------------------------------------------------------------------
-// VM templates — one per node, cloned from the downloaded cloud image.
-// Templates are not started and have no cloud-init config; that is applied
-// per-clone so each node gets its own hostname (derived from VM name).
-// ---------------------------------------------------------------------------
-
-const templateSettings = {
-  template: true,
-  started: false,
-  stopOnDestroy: true,
-  scsiHardware: "virtio-scsi-pci",
-  cpu: {
-    cores: 2,
-    sockets: 1,
-    type: "host",
-    numa: true,
-  },
-  memory: {
-    dedicated: 2048,
-    floating: 0,
-  },
-  networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
-  serialDevices: [{}],
-  vga: { type: "serial0" },
-  agent: { enabled: true },
-};
-
-const pveTemplate = new proxmox.VmLegacy(
-  "k3s-template-pve",
-  {
-    ...templateSettings,
-    nodeName: "pve",
-    name: "k3s-ubuntu-noble-template",
-    disks: [
-      {
-        interface: "scsi0",
-        datastoreId: pveNodes["pve-b550"].dataStore,
-        importFrom: pulumi.interpolate`${ubuntuImagePve.datastoreId}:import/${ubuntuImagePve.fileName}`,
-        size: 10,
-        ssd: true,
-        discard: "on",
-      },
-    ],
-  },
-  { provider: pveProvider, ignoreChanges: ["disks"] },
-);
-
-const pveBckpTemplate = new proxmox.VmLegacy(
-  "k3s-template-pve-bckp",
-  {
-    ...templateSettings,
-    nodeName: "pve-bckp",
-    name: "k3s-ubuntu-noble-template",
-    disks: [
-      {
-        interface: "scsi0",
-        datastoreId: pveNodes["pve-optiplex"].dataStore,
-        importFrom: pulumi.interpolate`${ubuntuImagePveBckp.datastoreId}:import/${ubuntuImagePveBckp.fileName}`,
-        size: 10,
-        ssd: true,
-        discard: "on",
-      },
-    ],
-  },
-  { provider: pveBckpProvider, ignoreChanges: ["disks"] },
-);
-
-const pveEsprimoTemplate = new proxmox.VmLegacy(
-  "k3s-template-pve-esprimo",
-  {
-    ...templateSettings,
-    nodeName: "pve",
-    name: "k3s-ubuntu-noble-template",
-    disks: [
-      {
-        interface: "scsi0",
-        datastoreId: pveNodes["pve-esprimo"].dataStore,
-        importFrom: pulumi.interpolate`${ubuntuImagePveEsprimo.datastoreId}:import/${ubuntuImagePveEsprimo.fileName}`,
-        size: 10,
-        ssd: true,
-        discard: "on",
-      },
-    ],
-  },
-  { provider: pveEsprimoProvider, ignoreChanges: ["disks"] },
-);
-
 // ---------------------------------------------------------------------------
 // k3s nodes — full clones of their respective templates
 // ---------------------------------------------------------------------------

 const k3sVmPassword = config.requireSecret("k3sVmPassword");
-const sshPvePublicKey = config.requireSecret("sshPvePublicKey");
+const pve1SshPublicKey = config.requireSecret("pve1SshPublicKey");
+const pve2SshPublicKey = config.requireSecret("pve2SshPublicKey");
+const pve3SshPublicKey = config.requireSecret("pve3SshPublicKey");

 interface NodeConfig {
  name: string;
  role: "master" | "worker";
-  nodeName: string;
-  provider: proxmox.Provider;
-  template: proxmox.VmLegacy;
-  diskDatastore: string;
+  host: (typeof pveNodes)[number]; // one row from the pveNodes chain
  ip: pulumi.Output<string>;
  longhornDiskSize?: number;
 }

+const hostFor = (label: string) => pveNodes.find((n) => n.hostLabel === label)!;
+
 const nodeConfigs: NodeConfig[] = [
  {
    name: "k3s-master-1",
    role: "master",
-    nodeName: pveNodes["pve-b550"].nodeName,
-    provider: pveProvider,
-    template: pveTemplate,
-    diskDatastore: pveNodes["pve-b550"].dataStore,
+    host: hostFor("pve-b550"),
    ip: master1Ip,
  },
  {
    name: "k3s-master-2",
    role: "master",
-    nodeName: pveNodes["pve-esprimo"].nodeName,
-    provider: pveEsprimoProvider,
-    template: pveTemplate,
-    diskDatastore: pveNodes["pve-esprimo"].dataStore,
+    host: hostFor("pve-esprimo"),
    ip: master2Ip,
  },
  {
    name: "k3s-worker-1",
    role: "worker",
-    nodeName: pveNodes["pve-b550"].nodeName,
-    provider: pveProvider,
-    template: pveTemplate,
-    diskDatastore: pveNodes["pve-b550"].dataStore,
+    host: hostFor("pve-b550"),
    ip: worker1Ip,
    longhornDiskSize: 50,
  },
  {
    name: "k3s-master-3",
    role: "master",
-    nodeName: pveNodes["pve-optiplex"].nodeName,
-    provider: pveBckpProvider,
-    template: pveBckpTemplate,
-    diskDatastore: pveNodes["pve-optiplex"].dataStore,
+    host: hostFor("pve-optiplex"),
    ip: master3Ip,
  },
  {
    name: "k3s-worker-2",
    role: "worker",
-    nodeName: pveNodes["pve-optiplex"].nodeName,
-    provider: pveBckpProvider,
-    template: pveBckpTemplate,
-    diskDatastore: pveNodes["pve-optiplex"].dataStore,
+    host: hostFor("pve-optiplex"),
    ip: worker2Ip,
    longhornDiskSize: 50,
  },
  {
    name: "k3s-worker-3",
    role: "worker",
-    nodeName: pveNodes["pve-esprimo"].nodeName,
-    provider: pveEsprimoProvider,
-    template: pveEsprimoTemplate,
-    diskDatastore: pveNodes["pve-esprimo"].dataStore,
+    host: hostFor("pve-esprimo"),
    ip: worker3Ip,
    longhornDiskSize: 50,
  },
@@ -292,14 +227,14 @@ const k3sVms = nodeConfigs.map(
    new proxmox.VmLegacy(
      node.name,
      {
-        nodeName: node.nodeName,
+        nodeName: node.host.nodeName,
        name: node.name,
        description: "k3s " + node.role + " node — managed by Pulumi",
        tags: ["k3s", node.role],
        clone: {
-          vmId: node.template.vmId,
+          vmId: node.host.pveTemplate.vmId,
          full: true,
-          datastoreId: node.diskDatastore,
+          datastoreId: node.host.dataStore,
        },
        cpu: {
          cores: 2,
@@ -314,7 +249,7 @@ const k3sVms = nodeConfigs.map(
        disks: [
          {
            interface: "scsi0",
-            datastoreId: node.diskDatastore,
+            datastoreId: node.host.dataStore,
            size: 35,
            ssd: true,
            discard: "on",
@@ -324,7 +259,7 @@ const k3sVms = nodeConfigs.map(
            : [
                {
                  interface: "scsi1",
-                  datastoreId: node.diskDatastore,
+                  datastoreId: node.host.dataStore,
                  size: node.longhornDiskSize,
                  ssd: true,
                  discard: "on",
@@ -332,13 +267,15 @@ const k3sVms = nodeConfigs.map(
              ]),
        ],
        initialization: {
-          datastoreId: node.diskDatastore,
+          datastoreId: node.host.dataStore,
          ipConfigs: [{ ipv4: { address: "dhcp" } }],
          userAccount: {
            username: "ubuntu",
            password: k3sVmPassword,
            keys: [
-              sshPvePublicKey.apply((k) => k.trim()),
+              pve1SshPublicKey.apply((k) => k.trim()),
+              pve2SshPublicKey.apply((k) => k.trim()),
+              pve3SshPublicKey.apply((k) => k.trim()),
              ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
            ],
          },
@@ -347,12 +284,13 @@ const k3sVms = nodeConfigs.map(
        scsiHardware: "virtio-scsi-pci",
        serialDevices: [{}],
        vga: { type: "serial0" },
-        agent: { enabled: true },
+        agent: { enabled: true, timeout: "30s" },
+        cdrom: { fileId: "none" },
        started: false,
        stopOnDestroy: true,
      },
      {
-        provider: node.provider,
+        provider: node.host.provider,
        retainOnDelete: true,
        ignoreChanges: ["clone", "started"],
      },
@@ -7,12 +7,12 @@ Bootstraps a k3s cluster on the Proxmox VMs created by `proxmox-infra`. Starts V
 ## How it works

 1. Starts all VMs via `POST /api2/json/nodes/{node}/qemu/{vmid}/status/start`
-2. Waits for port 22 to open on each VM (bash `/dev/tcp`)
-3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
-4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
-5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes
+2. Waits for port 22 to open on each VM (bash `/dev/tcp`), installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip> --tls-san <controlPlaneVipAddress>`
+3. Waits for `https://<controlPlaneVipAddress>:6443/healthz` to respond before anyone else joins, since every subsequent join uses the VIP, not `master1Ip`. The VIP itself is a pfSense IP Alias Virtual IP with HAProxy TCP-load-balancing 6443 across all three masters — set up manually, not managed by any Pulumi stack (see "pfSense one-time setup" below). This is deliberately *not* a second kube-vip/ARP-based VIP: `03-k8s-infra`'s existing kube-vip DaemonSet already handles Service/LoadBalancer VIPs (e.g. Traefik) separately, entirely inside the cluster, and API access is too critical to depend on the same ARP-failover mechanism
+4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes via `--server https://<controlPlaneVipAddress>:6443`
+5. Joins `k3s-worker-1`, `k3s-worker-2`, and `k3s-worker-3` as agent nodes via `K3S_URL=https://<controlPlaneVipAddress>:6443`
 6. Formats the workers' dedicated `scsi1` disk (ext4, if not already formatted) and mounts it at `/mnt/longhorn-extra` (via `/etc/fstab`) — this is the disk `03-k8s-infra` later registers as an extra Longhorn disk, to keep Longhorn off the root filesystem and avoid `DiskPressure`
-7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
+7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL to `controlPlaneVipAddress`, and exports it as the secret stack output `kubeconfig`

 VM IDs and the CI runner SSH private key are read automatically from the `proxmox-infra` stack output via StackReference — no manual setup needed for those.

@@ -28,14 +28,31 @@ pulumi config set master2Ip "192.168.1.x"
 pulumi config set master3Ip "192.168.1.x"
 pulumi config set worker1Ip "192.168.1.x"
 pulumi config set worker2Ip "192.168.1.x"
+pulumi config set worker3Ip "192.168.1.x"
+
+# Control-plane VIP — floating IP for API server access, fronted by pfSense
+# + HAProxy (see "pfSense one-time setup" below). Must be a free address
+# outside the router's DHCP pool.
+pulumi config set --secret controlPlaneVipAddress "192.168.1.147"
 ```

-Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`) are read automatically from `proxmox-infra` via StackReference — do **not** set them here.
+Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`, `pve3Endpoint`, `pve3ApiToken`) are read automatically from `proxmox-infra` via StackReference — do **not** set them here.

 ## Prerequisites

 `proxmox-infra` must be deployed first. On its first `pulumi up` it generates an ed25519 SSH keypair via `@pulumi/tls`, writes the public key into every VM's cloud-init, and exports the private key as a secret stack output. VMs must be re-provisioned (or have the public key manually added to `~/.ssh/authorized_keys`) before this stack can SSH in.

+## pfSense one-time setup (before deploying)
+
+The `@pulumi/pfsense` provider used by `01-proxmox-infra` doesn't expose Virtual IP or HAProxy resources (only DHCP static mappings, DNS overrides, and firewall aliases), so this is manual, not Pulumi-managed:
+
+1. Confirm `controlPlaneVipAddress` falls outside the DHCP pool (Services → DHCP Server → Range).
+2. Firewall → Virtual IPs → Add: Type = `IP Alias`, Interface = `LAN`, Address = `<controlPlaneVipAddress>/32`.
+3. System → Package Manager → install `haproxy` if not already installed.
+4. Services → HAProxy → Backend: add a pool (e.g. `k3s-apiservers`), mode `TCP`, health check `TCP check` (or `ssl-hello-chk`), servers = `<master1Ip>:6443`, `<master2Ip>:6443`, `<master3Ip>:6443`.
+5. Services → HAProxy → Frontend: add a frontend listening on `<controlPlaneVipAddress>:6443`, type `TCP` (not HTTP — this must stay an opaque passthrough so clients validate k3s's own TLS cert, not one terminated at pfSense), default backend = the pool from step 4.
+6. Services → HAProxy → Settings: enable HAProxy, apply.
+
 ## CI/CD

 Workflow: `../.gitea/workflows/02-deploy-k8s-bootstrap.yaml`. Secret: `K8S_BOOTSTRAP_PULUMI_DEV_YAML`. Deploy is manual (`workflow_dispatch`) — re-running this stack restarts the bootstrap sequence against live VMs.
@@ -3,12 +3,10 @@ import * as command from "@pulumi/command";

 const config = new pulumi.Config();

-//fetch credentials from proxmox-infra
 const infraRef = new pulumi.StackReference(
  `${pulumi.getOrganization()}/proxmox-infra/dev`,
 );

-// Proxmox API credentials — same as proxmox-infra stack
 const pve1Endpoint = infraRef.requireOutput(
  "pve1Endpoint",
 ) as pulumi.Output<string>;
@@ -21,6 +19,12 @@ const pve2Endpoint = infraRef.requireOutput(
 const pve2ApiToken = infraRef.requireOutput(
  "pve2ApiToken",
 ) as pulumi.Output<string>;
+const pve3Endpoint = infraRef.requireOutput(
+  "pve3Endpoint",
+) as pulumi.Output<string>;
+const pve3ApiToken = infraRef.requireOutput(
+  "pve3ApiToken",
+) as pulumi.Output<string>;

 // Node IPs — static DHCP leases set in the router
 const master1Ip = infraRef.requireOutput("master1Ip");
@@ -28,10 +32,18 @@ const master2Ip = infraRef.requireOutput("master2Ip");
 const master3Ip = infraRef.requireOutput("master3Ip");
 const worker1Ip = infraRef.requireOutput("worker1Ip");
 const worker2Ip = infraRef.requireOutput("worker2Ip");
+const worker3Ip = infraRef.requireOutput("worker3Ip");

 // Pre-shared k3s cluster token
 const k3sToken = config.requireSecret("k3sToken");

+// Control-plane VIP — floating IP for API server access, ARP-announced by
+// kube-vip on whichever master is alive. Added as a TLS SAN on master-1's
+// cert below, then used (instead of master1Ip) for every other node's join
+// URL and for the exported kubeconfig, so the cluster stays reachable
+// through the loss of any single master.
+const controlPlaneVipAddress = config.requireSecret("controlPlaneVipAddress");
+
 // VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
 const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<
  Record<string, number>
@@ -45,6 +57,7 @@ const master2VmId = vmIdsOutput.apply((ids) => String(ids.master2));
 const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3));
 const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1));
 const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2));
+const worker3VmId = vmIdsOutput.apply((ids) => String(ids.worker3));

 // SSH connection helper
 function conn(
@@ -83,6 +96,11 @@ const startWorker2 = new command.local.Command("start-worker-2", {
  triggers: [worker2VmId],
  interpreter: ["/bin/bash", "-c"],
 });
+const startWorker3 = new command.local.Command("start-worker-3", {
+  create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve3ApiToken}" "${pve3Endpoint}/api2/json/nodes/pve/qemu/${worker3VmId}/status/start" 2>/dev/null || true`,
+  triggers: [worker3VmId],
+  interpreter: ["/bin/bash", "-c"],
+});

 const allStarts = [
  startMaster1,
@@ -90,6 +108,7 @@ const allStarts = [
  startMaster3,
  startWorker1,
  startWorker2,
+  startWorker3,
 ];

 // ---------------------------------------------------------------------------
@@ -110,7 +129,7 @@ const installMaster1 = new command.remote.Command(
  "install-k3s-master-1",
  {
    connection: conn(master1Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --disable servicelb --node-name k3s-master-1`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --tls-san ${controlPlaneVipAddress} --disable servicelb --node-name k3s-master-1`,
    triggers: [master1VmId],
  },
  { dependsOn: [waitMaster1Ssh] },
@@ -127,7 +146,30 @@ const waitK3sMaster1Ready = new command.remote.Command(
 );

 // ---------------------------------------------------------------------------
-// Step 3: Wait for SSH on remaining masters, join cluster
+// Step 3: Confirm the control-plane VIP is actually reachable before anyone
+// else joins, so master-2/master-3 and every worker join through the VIP
+// instead of master1Ip directly. The VIP itself is NOT managed here (or by
+// any Pulumi stack) — it's a pfSense IP Alias Virtual IP with HAProxy
+// TCP-load-balancing 6443 across all three masters, set up manually (see
+// "pfSense one-time setup" in this stack's CLAUDE.md). This intentionally
+// avoids a second kube-vip/ARP-based VIP for something as critical as API
+// access — 03-k8s-infra's existing kube-vip DaemonSet already handles
+// Service/LoadBalancer VIPs (e.g. Traefik) separately, entirely inside the
+// cluster.
+// ---------------------------------------------------------------------------
+
+const waitControlPlaneVipReady = new command.remote.Command(
+  "wait-control-plane-vip-ready",
+  {
+    connection: conn(master1Ip),
+    create: pulumi.interpolate`for i in $(seq 1 60); do curl -sk --max-time 3 https://${controlPlaneVipAddress}:6443/healthz && exit 0; sleep 5; done; exit 1`,
+    triggers: [master1VmId],
+  },
+  { dependsOn: [waitK3sMaster1Ready] },
+);
+
+// ---------------------------------------------------------------------------
+// Step 4: Wait for SSH on remaining masters, join cluster via the VIP
 // ---------------------------------------------------------------------------

 const waitMaster2Ssh = new command.local.Command(
@@ -154,24 +196,24 @@ const joinMaster2 = new command.remote.Command(
  "join-k3s-master-2",
  {
    connection: conn(master2Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-2`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-2`,
    triggers: [master2VmId],
  },
-  { dependsOn: [waitMaster2Ssh] },
+  { dependsOn: [waitMaster2Ssh, waitControlPlaneVipReady] },
 );

 const joinMaster3 = new command.remote.Command(
  "join-k3s-master-3",
  {
    connection: conn(master3Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-3`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-3`,
    triggers: [master3VmId],
  },
-  { dependsOn: [waitMaster3Ssh, joinMaster2] },
+  { dependsOn: [waitMaster3Ssh, waitControlPlaneVipReady, joinMaster2] },
 );

 // ---------------------------------------------------------------------------
-// Step 4: Join worker nodes
+// Step 5: Join worker nodes via the VIP
 // ---------------------------------------------------------------------------

 const waitWorker1Ssh = new command.local.Command(
@@ -194,11 +236,21 @@ const waitWorker2Ssh = new command.local.Command(
  { dependsOn: [joinMaster3] },
 );

+const waitWorker3Ssh = new command.local.Command(
+  "wait-ssh-worker-3",
+  {
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    triggers: [worker3VmId],
+    interpreter: ["/bin/bash", "-c"],
+  },
+  { dependsOn: [joinMaster3] },
+);
+
 const joinWorker1 = new command.remote.Command(
  "join-k3s-worker-1",
  {
    connection: conn(worker1Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
    triggers: [worker1VmId],
  },
  { dependsOn: [waitWorker1Ssh] },
@@ -208,31 +260,41 @@ const joinWorker2 = new command.remote.Command(
  "join-k3s-worker-2",
  {
    connection: conn(worker2Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
    triggers: [worker2VmId],
  },
  { dependsOn: [waitWorker2Ssh] },
 );

+const joinWorker3 = new command.remote.Command(
+  "join-k3s-worker-3",
+  {
+    connection: conn(worker3Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`,
+    triggers: [worker3VmId],
+  },
+  { dependsOn: [waitWorker3Ssh] },
+);
+
 // ---------------------------------------------------------------------------
-// Step 5: Format and mount Longhorn disks
+// Step 6: Format and mount Longhorn disks
 // ---------------------------------------------------------------------------

-const workers = [worker1Ip, worker2Ip];
-const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh];
+const workers = [worker1Ip, worker2Ip, worker3Ip];
+const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh, waitWorker3Ssh];

-const mountLonghornDisks = workers.map((worker, i) => {
+workers.forEach((worker, i) => {
  return new command.remote.Command(
    `mount-longhorn-disk-${i}`,
    {
      connection: conn(worker),
-      create: `set -euo pipefail; 
-        DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1'); 
-        DISK_PATH="/dev/disk/by-id/$DISK_ID"; 
-        sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH"; 
-        sudo mkdir -p /mnt/longhorn-extra; 
-        echo ""$DISK_PATH" /mnt/longhorn-extra  ext4  defaults,nofail  0  2" | sudo tee -a /etc/fstab; 
-        sudo mount -a; 
+      create: `set -euo pipefail;
+        DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1');
+        DISK_PATH="/dev/disk/by-id/$DISK_ID";
+        sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH";
+        sudo mkdir -p /mnt/longhorn-extra;
+        echo ""$DISK_PATH" /mnt/longhorn-extra  ext4  defaults,nofail  0  2" | sudo tee -a /etc/fstab;
+        sudo mount -a;
        df -h /mnt/longhorn-extra`,
    },
    { dependsOn: workerSshReader[i] },
@@ -240,7 +302,7 @@ const mountLonghornDisks = workers.map((worker, i) => {
 });

 // ---------------------------------------------------------------------------
-// Step 6: Read kubeconfig from master-1, patch server URL for external access
+// Step 7: Read kubeconfig from master-1, patch server URL to the VIP
 // ---------------------------------------------------------------------------

 const getKubeconfig = new command.remote.Command(
@@ -250,11 +312,66 @@ const getKubeconfig = new command.remote.Command(
    create: `sudo cat /etc/rancher/k3s/k3s.yaml`,
    triggers: [master1VmId],
  },
-  { dependsOn: [joinWorker1, joinWorker2] },
+  { dependsOn: [joinWorker1, joinWorker2, joinWorker3] },
 );

+// ---------------------------------------------------------------------------
+// Step 8: Install additional packages
+// ---------------------------------------------------------------------------
+
+const allVms = [
+  {
+    id: master1VmId,
+    ip: master1Ip,
+    name: "master-1",
+    readiness: waitMaster1Ssh,
+  },
+  {
+    id: master2VmId,
+    ip: master2Ip,
+    name: "master-2",
+    readiness: waitMaster2Ssh,
+  },
+  {
+    id: master3VmId,
+    ip: master3Ip,
+    name: "master-3",
+    readiness: waitMaster3Ssh,
+  },
+  {
+    id: worker1VmId,
+    ip: worker1Ip,
+    name: "worker-1",
+    readiness: waitWorker1Ssh,
+  },
+  {
+    id: worker2VmId,
+    ip: worker2Ip,
+    name: "worker-2",
+    readiness: waitWorker2Ssh,
+  },
+  {
+    id: worker3VmId,
+    ip: worker3Ip,
+    name: "worker-3",
+    readiness: waitWorker3Ssh,
+  },
+];
+
+allVms.forEach(({ id, ip, name, readiness }) => {
+  new command.remote.Command(
+    `install-packages-${name}`,
+    {
+      connection: conn(ip),
+      create: pulumi.interpolate`sudo apt update && sudo apt install -y qemu-guest-agent && sudo systemctl enable --now qemu-guest-agent`,
+      triggers: [id],
+    },
+    { dependsOn: readiness },
+  );
+});
+
 export const kubeconfig = pulumi.secret(
  pulumi
-    .all([getKubeconfig.stdout, master1Ip])
-    .apply(([kc, ip]) => kc.replaceAll("127.0.0.1", ip as string).trim()),
+    .all([getKubeconfig.stdout, controlPlaneVipAddress])
+    .apply(([kc, vip]) => kc.replaceAll("127.0.0.1", vip as string).trim()),
 );
@@ -143,7 +143,7 @@ const longhorn = new k8s.helm.v3.Release(
 );

 // Patches Longhorn instances to use the mountend Longhorn storage on the worker nodes.
-["k3s-worker-1", "k3s-worker-2"].forEach(
+["k3s-worker-1", "k3s-worker-2", "k3s-worker-3"].forEach(
  (nodeName) =>
    new k8s.apiextensions.CustomResourcePatch(
      `${nodeName}-longhorn-disks`,
@@ -0,0 +1,63 @@
+# Git Conventions
+
+## Branch Naming
+
+```
+<type>/<short-kebab-case-description>
+```
+
+- All lowercase, hyphens only — no camelCase or Title-Case
+- Keep the description short (3–5 words); the branch name is not the place for detail
+- No ticket tracker is used on this project, so branch names don't carry an ID
+
+| Type | Use for |
+|---|---|
+| `feature/` | New functionality (e.g. a new stack, a new resource type) |
+| `bug/` | Bug fixes |
+| `enhancement/` | Improvements or refactors to existing functionality, no new capability |
+| `chore/` | Dependency bumps, tooling, CI/CD, cleanup |
+| `docs/` | Documentation only (README, CLAUDE.md files) |
+
+**Examples:** `feature/add-3rd-proxmox-node`, `bug/fix-longhorn-disk-pressure`, `enhancement/refactor-k8s-bootstrap`
+
+---
+
+## Commit Messages
+
+```
+<type>: <imperative, present-tense summary>
+
+<optional body — explain WHY, not WHAT>
+```
+
+- **Type** — same list as branch types above (`feature`, `bug`, `enhancement`, `chore`, `docs`)
+- **Scope is optional** and, if used, should be one of the stack directories this repo is organized into — `proxmox-infra`, `k8s-bootstrap`, `k8s-infra`, `k8s-apps`, `monitoring` — or `ci` for `.gitea/workflows` changes: `fix(k8s-infra): ...`
+- **Summary** — imperative mood ("add", not "added" or "adds"); no period at the end
+- **Body** — only when the reasoning isn't obvious from the diff (a constraint, a workaround for a Proxmox/k3s quirk, a decision between StackReference outputs). Skip it for simple/self-explanatory changes.
+
+**Examples:**
+```
+feature: add pfsense provider for automating static IP setup
+bug: prune stale kube-vip leases after node replacement
+chore: upgrade Pulumi provider packages across all stacks
+docs: update CLAUDE.md with new Longhorn mount settings
+enhancement(k8s-bootstrap): cache dependencies to speed up deployment
+```
+
+---
+
+## Pull Request Titles
+
+This repo uses Gitea Actions (see root `CLAUDE.md`); Gitea's merge commit reuses the PR title verbatim as the merge summary, so the title should stand on its own in `git log`. Frame it as the outcome, not a task log:
+
+```
+<Type>: <what changes for the cluster/infra>
+```
+
+**Examples:**
+- `Feature: Add 3rd Proxmox node to the cluster`
+- `Bug: Fix Longhorn disk pressure from root disk usage`
+- `Enhancement: Refactor k8s-bootstrap for clearer deployment order`
+- `Chore: Upgrade Pulumi packages across all stacks`
+
+Since `01`/`02` deploys are manual (`workflow_dispatch`) and `03`–`05` deploy on merge to `main`, a clear PR title also doubles as a quick changelog of what just went live.
Author	SHA1	Message	Date
kasun	19bf92d6bc	docs(k8s-bootstrap): document control-plane VIP and pfSense HAProxy setup Deploy k8s Bootstrap / Bootstrap k3s Cluster (pull_request) Skipped Details Deploy k8s Bootstrap / Pulumi Preview (pull_request) Successful in 40s Details The VIP and HAProxy frontend/backend aren't exposed by the pfsense provider, so they must be created by hand before this stack is deployed.	2026-10-02 23:52:32 +02:00
kasun	afac1f338f	feature(k8s-bootstrap): join nodes and export kubeconfig via control-plane VIP Every join and the exported kubeconfig previously pointed at master1Ip, so losing master-1 cut off API access even with etcd quorum intact. Use controlPlaneVipAddress (a pfSense IP Alias fronted by HAProxy across all three masters) instead, add it as a TLS SAN on master-1, and wait for the VIP's /healthz before any other node joins. The VIP is set up manually on pfSense rather than via a second kube-vip/ARP VIP: API access is too critical to depend on the same failover mechanism that serves LoadBalancer Services. Also strips trailing whitespace from the Longhorn disk mount script.	2026-10-02 23:52:32 +02:00
kasun	2f57f1822f	Merge pull request 'Enhancement/consolidate proxmox node config' (#32 ) from enhancement/consolidate-proxmox-node-config into main Reviewed-on: #32	2026-08-05 16:39:09 +02:00
kasun	07c4678a6d	docs(proxmox-infra): removed inline comments Deploy k8s Bootstrap / Pulumi Preview (pull_request) Successful in 48s Details Deploy k8s Bootstrap / Bootstrap k3s Cluster (pull_request) Skipped Details Deploy Proxmox Infra / Pulumi Deploy (pull_request) Skipped Details Deploy Proxmox Infra / Pulumi Preview (pull_request) Successful in 1m22s Details	2026-08-05 16:32:07 +02:00
kasun	fb1f8db36b	enhancenment(proxmox-infra): added ssh pub keys of the other 2 nodes to cloud init vm creation. added the nodes manually since already created. Deploy k8s Bootstrap / Pulumi Preview (pull_request) Successful in 39s Details Deploy k8s Bootstrap / Bootstrap k3s Cluster (pull_request) Skipped Details Deploy Proxmox Infra / Pulumi Deploy (pull_request) Skipped Details Deploy Proxmox Infra / Pulumi Preview (pull_request) Successful in 1m16s Details	2026-08-05 16:31:38 +02:00
kasun	0ae688ac66	enhancement(proxmox-infra): consolidate per-host config into a single table Deploy Proxmox Infra / Pulumi Preview (pull_request) Successful in 53s Details Deploy Proxmox Infra / Pulumi Deploy (pull_request) Skipped Details Deploy k8s Bootstrap / Pulumi Preview (pull_request) Successful in 40s Details Deploy k8s Bootstrap / Bootstrap k3s Cluster (pull_request) Skipped Details Provider, image download, and template were three separate consts per physical node with matching lookups scattered through nodeConfigs - easy to update one and miss another when adding a node. Folded them into one pveNodes pipeline (seed data -> provider -> image -> template) and had nodeConfigs reference a host row instead of copying its fields. Also folds in two fixes discovered while bringing up worker-3, now inseparable from the restructuring above: - disable the default physical CD-ROM passthrough (cdrom: fileId "none") that failed VM start on nodes without an optical drive - shorten the qemu-agent timeout from the 15m default so refresh/read fails fast instead of hanging	2026-07-30 02:55:01 +02:00
kasun	b7d5a5c5cd	bug(k8s-bootstrap): install and enable qemu-guest-agent on all nodes qemu-guest-agent isn't present in the stock Ubuntu Noble cloud image, which was causing pulumi refresh on proxmox-infra to block up to 15m per VM waiting for agent data that never arrives.	2026-07-30 02:54:56 +02:00
kasun	10a4578ce4	Merge pull request 'feature: join k3s-worker-3 into the cluster' (#31 ) from feature/add-3rd-k8s-worker into main Deploy k8s Infra / Pulumi Preview (push) Skipped Details Deploy k8s Infra / Pulumi Deploy (push) Successful in 1m0s Details Reviewed-on: #31	2026-07-28 22:56:50 +02:00
kasun	3f98874008	feature: join k3s-worker-3 into the cluster Deploy k8s Bootstrap / Pulumi Preview (pull_request) Successful in 45s Details Deploy k8s Bootstrap / Bootstrap k3s Cluster (pull_request) Skipped Details Deploy k8s Infra / Pulumi Preview (pull_request) Successful in 47s Details Deploy k8s Infra / Pulumi Deploy (pull_request) Skipped Details Wires the k8s-bootstrap join sequence (start, SSH wait, k3s agent join, Longhorn disk mount) and the k8s-infra Longhorn disk patch for worker-3. Its VM was already provisioned by proxmox-infra in an earlier branch.	2026-07-28 22:52:22 +02:00
kasun	e830eb3b40	docs: add git conventions guide	2026-07-28 22:38:12 +02:00