fix: added shutdown logic to deployment yaml to speed up deployment runs

This reverts commit 354cf65657.

.env.local

@@ -1,3 +0,0 @@
-GITEA_API_URL=https://gitea.kasuns.website/kasun/homelab-infrastructure-as-code.git
-GITEA_TOKEN=ba3fd0f4851aa627e2088da1f94a596646ba2de7
-

.gitea/workflows/01-deploy-proxmox-infra.yaml

@@ -0,0 +1,142 @@
+name: Deploy Proxmox Infra
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - '01-proxmox-infra/**'
+      - '.gitea/workflows/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '01-proxmox-infra/**'
+      - '.gitea/workflows/**'
+
+jobs:
+  preview:
+    name: Pulumi Preview
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 01-proxmox-infra/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm ci
+        working-directory: 01-proxmox-infra
+
+      - name: Install Pulumi CLI
+        run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
+
+      - name: Generate Local pfSense SDK
+        run: pulumi package add terraform-provider marshallford/pfsense 0.22.0
+        working-directory: 01-proxmox-infra
+
+      - name: Preview
+        uses: pulumi/actions@v5
+        with:
+          command: preview
+          stack-name: dev
+          work-dir: 01-proxmox-infra
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+  deploy:
+    name: Pulumi Deploy
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 01-proxmox-infra/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm ci
+        working-directory: 01-proxmox-infra
+
+      - name: Install Pulumi CLI
+        run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
+
+      - name: Generate Local pfSense SDK
+        run: pulumi package add terraform-provider marshallford/pfsense 0.22.0
+        working-directory: 01-proxmox-infra
+
+      - name: Shutdown VMs
+        run: |
+          pulumi login "$PULUMI_BACKEND_URL"
+          PVE1=$(pulumi stack output --stack dev --show-secrets pve1Endpoint)
+          TOKEN1=$(pulumi stack output --stack dev --show-secrets pve1ApiToken)
+          PVE2=$(pulumi stack output --stack dev --show-secrets pve2Endpoint)
+          TOKEN2=$(pulumi stack output --stack dev --show-secrets pve2ApiToken)
+          IDS=$(pulumi stack output --stack dev --json vmIds)
+          M1=$(echo "$IDS" | jq -r .master1)
+          M2=$(echo "$IDS" | jq -r .master2)
+          W1=$(echo "$IDS" | jq -r .worker1)
+          M3=$(echo "$IDS" | jq -r .master3)
+          W2=$(echo "$IDS" | jq -r .worker2)
+
+          for id in $M1 $M2 $W1; do
+            curl -sf -k -X POST "$PVE1/api2/json/nodes/pve/qemu/$id/status/shutdown" \
+              -H "Authorization: PVEAPIToken=$TOKEN1" || true
+          done
+          for id in $M3 $W2; do
+            curl -sf -k -X POST "$PVE2/api2/json/nodes/pve-bckp/qemu/$id/status/shutdown" \
+              -H "Authorization: PVEAPIToken=$TOKEN2" || true
+          done
+
+          wait_stopped() {
+            local ep=$1 tok=$2 node=$3 id=$4
+            for i in $(seq 1 36); do
+              status=$(curl -sf -k "$ep/api2/json/nodes/$node/qemu/$id/status/current" \
+                -H "Authorization: PVEAPIToken=$tok" | jq -r .data.status)
+              [ "$status" = "stopped" ] && return 0
+              sleep 5
+            done
+            echo "Timeout: VM $id did not stop within 3 minutes" && exit 1
+          }
+
+          for id in $M1 $M2 $W1; do wait_stopped "$PVE1" "$TOKEN1" pve "$id"; done
+          for id in $M3 $W2; do wait_stopped "$PVE2" "$TOKEN2" pve-bckp "$id"; done
+        working-directory: 01-proxmox-infra
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+          PULUMI_BACKEND_URL: ${{ secrets.PULUMI_BACKEND_URL }}
+
+      - name: Refresh State
+        uses: pulumi/actions@v5
+        with:
+          command: refresh
+          stack-name: dev
+          work-dir: 01-proxmox-infra
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+      - name: Deploy
+        uses: pulumi/actions@v5
+        with:
+          command: up
+          stack-name: dev
+          work-dir: 01-proxmox-infra
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

.gitea/workflows/02-deploy-k8s-bootstrap.yaml

@@ -6,14 +6,14 @@ on:
     branches:
       - main
     paths:
-      - 'k8s-bootstrap/**'
-      - '.gitea/workflows/deploy-k8s-bootstrap.yaml'
+      - "02-k8s-bootstrap/**"
+      - ".gitea/workflows/02-deploy-k8s-bootstrap.yaml"
   pull_request:
     branches:
       - main
     paths:
-      - 'k8s-bootstrap/**'
-      - '.gitea/workflows/deploy-k8s-bootstrap.yaml'
+      - "02-k8s-bootstrap/**"
+      - ".gitea/workflows/02-deploy-k8s-bootstrap.yaml"
 
 jobs:
   preview:
@@ -27,21 +27,21 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '24'
+          node-version: "24"
 
       - name: Restore Stack Config
-        run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
+        run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > 02-k8s-bootstrap/Pulumi.dev.yaml
 
       - name: Install Dependencies
         run: npm install
-        working-directory: k8s-bootstrap
+        working-directory: 02-k8s-bootstrap
 
       - name: Preview
         uses: pulumi/actions@v5
         with:
           command: preview
           stack-name: dev
-          work-dir: k8s-bootstrap
+          work-dir: 02-k8s-bootstrap
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
@@ -57,21 +57,21 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '24'
+          node-version: "24"
 
       - name: Restore Stack Config
-        run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
+        run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > 02-k8s-bootstrap/Pulumi.dev.yaml
 
       - name: Install Dependencies
         run: npm install
-        working-directory: k8s-bootstrap
+        working-directory: 02-k8s-bootstrap
 
       - name: Refresh State
         uses: pulumi/actions@v5
         with:
           command: refresh
           stack-name: dev
-          work-dir: k8s-bootstrap
+          work-dir: 02-k8s-bootstrap
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
@@ -81,7 +81,7 @@ jobs:
         with:
           command: up
           stack-name: dev
-          work-dir: k8s-bootstrap
+          work-dir: 02-k8s-bootstrap
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

.gitea/workflows/03-deploy-k8s-infra.yaml

@@ -1,4 +1,4 @@
-name: Deploy Proxmox Infra
+name: Deploy k8s Infra
 
 on:
   workflow_dispatch:
@@ -6,13 +6,13 @@ on:
     branches:
       - main
     paths:
-      - 'proxmox-infra/**'
+      - '03-k8s-infra/**'
       - '.gitea/workflows/**'
   pull_request:
     branches:
       - main
     paths:
-      - 'proxmox-infra/**'
+      - '03-k8s-infra/**'
       - '.gitea/workflows/**'
 
 jobs:
@@ -30,18 +30,21 @@ jobs:
           node-version: '24'
 
       - name: Restore Stack Config
-        run: echo "${{ secrets.PULUMI_DEV_YAML }}" | base64 -d > proxmox-infra/Pulumi.dev.yaml
+        run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 03-k8s-infra/Pulumi.dev.yaml
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
 
       - name: Install Dependencies
-        run: npm install
-        working-directory: proxmox-infra
+        run: npm ci
+        working-directory: 03-k8s-infra
 
       - name: Preview
         uses: pulumi/actions@v5
         with:
           command: preview
           stack-name: dev
-          work-dir: proxmox-infra
+          work-dir: 03-k8s-infra
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
@@ -60,18 +63,21 @@ jobs:
           node-version: '24'
 
       - name: Restore Stack Config
-        run: echo "${{ secrets.PULUMI_DEV_YAML }}" | base64 -d > proxmox-infra/Pulumi.dev.yaml
+        run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 03-k8s-infra/Pulumi.dev.yaml
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
 
       - name: Install Dependencies
-        run: npm install
-        working-directory: proxmox-infra
+        run: npm ci
+        working-directory: 03-k8s-infra
 
       - name: Refresh State
         uses: pulumi/actions@v5
         with:
           command: refresh
           stack-name: dev
-          work-dir: proxmox-infra
+          work-dir: 03-k8s-infra
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
@@ -81,7 +87,7 @@ jobs:
         with:
           command: up
           stack-name: dev
-          work-dir: proxmox-infra
+          work-dir: 03-k8s-infra
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

.gitignore

@@ -3,4 +3,6 @@
 .vscode
 node_modules/
 bin/
-Pulumi.dev.yaml
+Pulumi.dev.yaml
+sdks/
+.env

01-proxmox-infra/CLAUDE.md

@@ -16,9 +16,11 @@ npm install
 pulumi preview
 
 # Sync Pulumi state with actual Proxmox state (run before up if resources were changed manually)
+# Note: shut down all k3s VMs in Proxmox first — refresh is slow against running VMs
 pulumi refresh --yes
 
 # Deploy infrastructure
+# Note: shut down all k3s VMs in Proxmox first, then run:
 pulumi refresh --yes && pulumi up --yes
 
 # Destroy infrastructure

01-proxmox-infra/Pulumi.yaml

@@ -8,3 +8,9 @@ config:
   pulumi:tags:
     value:
       pulumi:template: typescript
+packages:
+  pfsense:
+    source: terraform-provider
+    version: 1.1.3
+    parameters:
+      - marshallford/pfsense

01-proxmox-infra/index.ts

@@ -1,18 +1,29 @@
 import * as pulumi from "@pulumi/pulumi";
 import * as proxmox from "@muhlba91/pulumi-proxmoxve";
 import * as tls from "@pulumi/tls";
+import * as pfsense from "@pulumi/pfsense";
 
 const config = new pulumi.Config();
 
-// ---------------------------------------------------------------------------
-// Providers — one per standalone Proxmox machine
-// ---------------------------------------------------------------------------
-
 const pve1Endpoint = config.requireSecret("pve1Endpoint");
 const pve1ApiToken = config.requireSecret("pve1ApiToken");
 const pve2Endpoint = config.requireSecret("pve2Endpoint");
 const pve2ApiToken = config.requireSecret("pve2ApiToken");
 
+const pfSenseUrl = config.requireSecret("pfSenseUrl");
+const pfSenseUser = config.requireSecret("pfSenseUser");
+const pfSensePassword = config.requireSecret("pfSensePassword");
+
+const master1Ip = config.requireSecret("master1Ip");
+const master2Ip = config.requireSecret("master2Ip");
+const worker1Ip = config.requireSecret("worker1Ip");
+const master3Ip = config.requireSecret("master3Ip");
+const worker2Ip = config.requireSecret("worker2Ip");
+
+// ---------------------------------------------------------------------------
+// Providers — one per standalone Proxmox machine
+// ---------------------------------------------------------------------------
+
 const pveProvider = new proxmox.Provider("pve", {
   endpoint: pve1Endpoint,
   apiToken: pve1ApiToken,
@@ -25,6 +36,17 @@ const pveBckpProvider = new proxmox.Provider("pve-bckp", {
   insecure: true,
 });
 
+// ---------------------------------------------------------------------------
+// Providers — PfSense
+// ---------------------------------------------------------------------------
+
+const pfSenseProvider = new pfsense.Provider("pfsense", {
+  url: pfSenseUrl,
+  username: pfSenseUser,
+  password: pfSensePassword,
+  tlsSkipVerify: true,
+});
+
 // ---------------------------------------------------------------------------
 // CI runner SSH keypair — generated once, stored in Pulumi state backend.
 // Public key goes into every VM; private key is exported for k8s-bootstrap.
@@ -49,7 +71,7 @@ const ubuntuImagePve = new proxmox.download.File(
     contentType: "import",
     fileName: "noble-server-cloudimg-amd64.qcow2",
     url: ubuntuNobleUrl,
-    overwrite: true,
+    overwrite: false,
     overwriteUnmanaged: true,
   },
   { provider: pveProvider },
@@ -63,7 +85,7 @@ const ubuntuImagePveBckp = new proxmox.download.File(
     contentType: "import",
     fileName: "noble-server-cloudimg-amd64.qcow2",
     url: ubuntuNobleUrl,
-    overwrite: true,
+    overwrite: false,
     overwriteUnmanaged: true,
   },
   { provider: pveBckpProvider },
@@ -113,7 +135,7 @@ const pveTemplate = new proxmox.VmLegacy(
       },
     ],
   },
-  { provider: pveProvider },
+  { provider: pveProvider, ignoreChanges: ["disks"] },
 );
 
 const pveBckpTemplate = new proxmox.VmLegacy(
@@ -133,7 +155,7 @@ const pveBckpTemplate = new proxmox.VmLegacy(
       },
     ],
   },
-  { provider: pveBckpProvider },
+  { provider: pveBckpProvider, ignoreChanges: ["disks"] },
 );
 
 // ---------------------------------------------------------------------------
@@ -150,6 +172,7 @@ interface NodeConfig {
   provider: proxmox.Provider;
   template: proxmox.VmLegacy;
   diskDatastore: string;
+  ip: pulumi.Output<string>;
 }
 
 const nodeConfigs: NodeConfig[] = [
@@ -160,6 +183,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveProvider,
     template: pveTemplate,
     diskDatastore: "local-lvm",
+    ip: master1Ip,
   },
   {
     name: "k3s-master-2",
@@ -168,6 +192,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveProvider,
     template: pveTemplate,
     diskDatastore: "local-lvm",
+    ip: master2Ip,
   },
   {
     name: "k3s-worker-1",
@@ -176,6 +201,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveProvider,
     template: pveTemplate,
     diskDatastore: "local-lvm",
+    ip: worker1Ip,
   },
   {
     name: "k3s-master-3",
@@ -184,6 +210,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveBckpProvider,
     template: pveBckpTemplate,
     diskDatastore: "local",
+    ip: master3Ip,
   },
   {
     name: "k3s-worker-2",
@@ -192,6 +219,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveBckpProvider,
     template: pveBckpTemplate,
     diskDatastore: "local",
+    ip: worker2Ip,
   },
 ];
 
@@ -235,9 +263,9 @@ const k3sVms = nodeConfigs.map(
             username: "ubuntu",
             password: k3sVmPassword,
             keys: [
-                  sshPvePublicKey.apply((k) => k.trim()),
-                  ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
-                ],
+              sshPvePublicKey.apply((k) => k.trim()),
+              ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
+            ],
           },
         },
         networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
@@ -251,17 +279,27 @@ const k3sVms = nodeConfigs.map(
       {
         provider: node.provider,
         retainOnDelete: true,
-        ignoreChanges: ["clone"],
+        ignoreChanges: ["clone", "started", "disks"],
       },
     ),
 );
 
-export const clusterInfo = k3sVms.map((vm, index) => ({
-  nodeName: vm.nodeName,
-  vmId: vm.vmId,
-  name: nodeConfigs[index].name,
-  role: nodeConfigs[index].role,
-}));
+k3sVms.forEach((vmResource, i) => {
+  const assignedMac = vmResource.networkDevices.apply(
+    (nic) => nic[0].macAddress,
+  );
+
+  return new pfsense.Dhcpv4Staticmapping(
+    `${nodeConfigs[i].name}-dhcp`,
+    {
+      interface: "lan",
+      macAddress: assignedMac,
+      ipAddress: nodeConfigs[i].ip,
+      hostname: nodeConfigs[i].name,
+    },
+    { dependsOn: vmResource, provider: pfSenseProvider },
+  );
+});
 
 // Individual vmId exports — used by k8s-bootstrap to start VMs.
 // Order matches nodeConfigs: master-1, master-2, worker-1, master-3, worker-2.
@@ -278,3 +316,6 @@ export const ciRunnerPrivateKey = pulumi.secret(ciRunnerKey.privateKeyOpenssh);
 
 // Proxmox API credentials — consumed by k8s-bootstrap via StackReference.
 export { pve1Endpoint, pve1ApiToken, pve2Endpoint, pve2ApiToken };
+
+//k3s instance ips consumed by k8s-bootstrap.
+export { master1Ip, master2Ip, worker1Ip, master3Ip, worker2Ip };

01-proxmox-infra/package-lock.json

@@ -7,6 +7,7 @@
             "name": "proxmox-infra",
             "dependencies": {
                 "@muhlba91/pulumi-proxmoxve": "^8.2.1",
+                "@pulumi/pfsense": "file:sdks/pfsense",
                 "@pulumi/pulumi": "^3.113.0",
                 "@pulumi/tls": "^5.5.0"
             },
@@ -703,6 +704,10 @@
             "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==",
             "license": "BSD-3-Clause"
         },
+        "node_modules/@pulumi/pfsense": {
+            "resolved": "sdks/pfsense",
+            "link": true
+        },
         "node_modules/@pulumi/pulumi": {
             "version": "3.243.0",
             "resolved": "https://registry.npmjs.org/@pulumi/pulumi/-/pulumi-3.243.0.tgz",
@@ -2674,6 +2679,44 @@
             "engines": {
                 "node": ">=12"
             }
+        },
+        "sdks/pfsense": {
+            "name": "@pulumi/pfsense",
+            "version": "0.22.0",
+            "hasInstallScript": true,
+            "dependencies": {
+                "@pulumi/pulumi": "^3.238.0",
+                "@types/node": "^20",
+                "typescript": "^4.7.0"
+            }
+        },
+        "sdks/pfsense/node_modules/@types/node": {
+            "version": "20.19.41",
+            "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.41.tgz",
+            "integrity": "sha512-ECymXOukMnOoVkC2bb1Vc/w/836DXncOg5m8Xj1RH7xSHZJWNYY6Zh7EH477vcnD5egKNNfy2RpNOmuChhFPgQ==",
+            "license": "MIT",
+            "dependencies": {
+                "undici-types": "~6.21.0"
+            }
+        },
+        "sdks/pfsense/node_modules/typescript": {
+            "version": "4.9.5",
+            "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
+            "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+            "license": "Apache-2.0",
+            "bin": {
+                "tsc": "bin/tsc",
+                "tsserver": "bin/tsserver"
+            },
+            "engines": {
+                "node": ">=4.2.0"
+            }
+        },
+        "sdks/pfsense/node_modules/undici-types": {
+            "version": "6.21.0",
+            "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
+            "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+            "license": "MIT"
         }
     }
 }

01-proxmox-infra/package.json

@@ -7,7 +7,12 @@
     },
     "dependencies": {
         "@muhlba91/pulumi-proxmoxve": "^8.2.1",
+        "@pulumi/pfsense": "file:sdks/pfsense",
         "@pulumi/pulumi": "^3.113.0",
         "@pulumi/tls": "^5.5.0"
+    },
+    "imports": {
+    "#pfsense": "./sdks/pfsense/index.js",
+    "#pfsense/*": "./sdks/pfsense/*"
     }
 }

02-k8s-bootstrap/index.ts

@@ -23,11 +23,11 @@ const pve2ApiToken = infraRef.requireOutput(
 ) as pulumi.Output<string>;
 
 // Node IPs — static DHCP leases set in the router
-const master1Ip = config.require("master1Ip");
-const master2Ip = config.require("master2Ip");
-const master3Ip = config.require("master3Ip");
-const worker1Ip = config.require("worker1Ip");
-const worker2Ip = config.require("worker2Ip");
+const master1Ip = infraRef.requireOutput("master1Ip");
+const master2Ip = infraRef.requireOutput("master2Ip");
+const master3Ip = infraRef.requireOutput("master3Ip");
+const worker1Ip = infraRef.requireOutput("worker1Ip");
+const worker2Ip = infraRef.requireOutput("worker2Ip");
 
 // Pre-shared k3s cluster token
 const k3sToken = config.requireSecret("k3sToken");
@@ -47,7 +47,9 @@ const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1));
 const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2));
 
 // SSH connection helper
-function conn(ip: string): command.types.input.remote.ConnectionArgs {
+function conn(
+  ip: pulumi.Input<string>,
+): command.types.input.remote.ConnectionArgs {
   return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
 }
 
@@ -97,7 +99,7 @@ const allStarts = [
 const waitMaster1Ssh = new command.local.Command(
   "wait-ssh-master-1",
   {
-    create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
     triggers: [master1VmId],
     interpreter: ["/bin/bash", "-c"],
   },
@@ -131,7 +133,7 @@ const waitK3sMaster1Ready = new command.remote.Command(
 const waitMaster2Ssh = new command.local.Command(
   "wait-ssh-master-2",
   {
-    create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
     triggers: [master2VmId],
     interpreter: ["/bin/bash", "-c"],
   },
@@ -141,7 +143,7 @@ const waitMaster2Ssh = new command.local.Command(
 const waitMaster3Ssh = new command.local.Command(
   "wait-ssh-master-3",
   {
-    create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
     triggers: [master3VmId],
     interpreter: ["/bin/bash", "-c"],
   },
@@ -175,7 +177,7 @@ const joinMaster3 = new command.remote.Command(
 const waitWorker1Ssh = new command.local.Command(
   "wait-ssh-worker-1",
   {
-    create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
     triggers: [worker1VmId],
     interpreter: ["/bin/bash", "-c"],
   },
@@ -185,7 +187,7 @@ const waitWorker1Ssh = new command.local.Command(
 const waitWorker2Ssh = new command.local.Command(
   "wait-ssh-worker-2",
   {
-    create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
     triggers: [worker2VmId],
     interpreter: ["/bin/bash", "-c"],
   },
@@ -227,7 +229,7 @@ const getKubeconfig = new command.remote.Command(
 );
 
 export const kubeconfig = pulumi.secret(
-  getKubeconfig.stdout.apply((kc) =>
-    kc.replace(/127\.0\.0\.1/g, master1Ip).trim(),
-  ),
+  pulumi
+    .all([getKubeconfig.stdout, master1Ip])
+    .apply(([kc, ip]) => kc.replaceAll("127.0.0.1", ip as string).trim()),
 );

02-k8s-bootstrap/tsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "strict": true,
         "outDir": "bin",
-        "target": "es2020",
+        "target": "es2024",
         "module": "nodenext",
         "moduleResolution": "nodenext",
         "sourceMap": true,

03-k8s-infra/Pulumi.yaml

@@ -0,0 +1,6 @@
+name: k8s-infra
+description: Cluster-level infrastructure for the k3s homelab cluster
+runtime:
+  name: nodejs
+  options:
+    packagemanager: npm

03-k8s-infra/index.ts

@@ -0,0 +1,118 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as k8s from "@pulumi/kubernetes";
+
+const config = new pulumi.Config();
+
+//fetch credentials from k8s-bootstrap
+const infraRef = new pulumi.StackReference(
+  `${pulumi.getOrganization()}/k8s-bootstrap/dev`,
+);
+
+const kubeconfig = infraRef.requireOutput("kubeconfig");
+const truenasHost = config.requireSecret("truenasHost");
+const truenasNfsPath = config.requireSecret("truenasNfsPath");
+const cloudflareToken = config.requireSecret("cloudflareApiToken");
+const letsencryptEmail = config.requireSecret("letsencryptEmail");
+
+const k8sProvider = new k8s.Provider("k3s", { kubeconfig });
+const opts = (extras?: pulumi.ResourceOptions): pulumi.ResourceOptions => ({
+  provider: k8sProvider,
+  ...extras,
+});
+
+// ── 1. NFS CSI Driver ────────────────────────────────────────────────────────
+
+const nfsCsiDriver = new k8s.helm.v3.Release(
+  "nfs-csi-driver",
+  {
+    name: "csi-driver-nfs",
+    chart: "csi-driver-nfs",
+    repositoryOpts: {
+      repo: "https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts",
+    },
+    namespace: "kube-system",
+    version: "4.12.0",
+    values: {
+      kubeletDir: "/var/lib/kubelet",
+    },
+  },
+  opts(),
+);
+
+new k8s.storage.v1.StorageClass(
+  "truenas-nfs",
+  {
+    metadata: { name: "truenas-nfs" },
+    provisioner: "nfs.csi.k8s.io",
+    parameters: {
+      server: truenasHost,
+      share: truenasNfsPath,
+      mountPermissions: "0",
+    },
+    reclaimPolicy: "Retain",
+    volumeBindingMode: "Immediate",
+    allowVolumeExpansion: true,
+    mountOptions: ["nfsvers=4.1"],
+  },
+  opts({ dependsOn: [nfsCsiDriver] }),
+);
+
+// ── 2. cert-manager ──────────────────────────────────────────────────────────
+
+const certManager = new k8s.helm.v3.Release(
+  "cert-manager",
+  {
+    name: "cert-manager",
+    chart: "cert-manager",
+    repositoryOpts: { repo: "https://charts.jetstack.io" },
+    namespace: "cert-manager",
+    createNamespace: true,
+    timeout: 600,
+    values: {
+      crds: { enabled: true },
+    },
+  },
+  opts(),
+);
+
+// ── 3. Cloudflare token secret + ClusterIssuer ───────────────────────────────
+
+const cfSecret = new k8s.core.v1.Secret(
+  "cloudflare-token",
+  {
+    metadata: { name: "cloudflare-api-token", namespace: "cert-manager" },
+    stringData: { "api-token": cloudflareToken },
+  },
+  opts({ dependsOn: [certManager] }),
+);
+
+new k8s.apiextensions.CustomResource(
+  "letsencrypt-prod",
+  {
+    apiVersion: "cert-manager.io/v1",
+    kind: "ClusterIssuer",
+    metadata: { name: "letsencrypt-prod" },
+    spec: {
+      acme: {
+        server: "https://acme-v02.api.letsencrypt.org/directory",
+        email: letsencryptEmail,
+        privateKeySecretRef: { name: "letsencrypt-prod-account-key" },
+        solvers: [
+          {
+            dns01: {
+              cloudflare: {
+                apiTokenSecretRef: {
+                  name: cfSecret.metadata.name,
+                  key: "api-token",
+                },
+              },
+            },
+          },
+        ],
+      },
+    },
+  },
+  opts({ dependsOn: [certManager, cfSecret] }),
+);
+
+export const storageClass = "truenas-nfs";

03-k8s-infra/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "k8s-infra",
+    "main": "index.ts",
+    "devDependencies": {
+        "@types/node": "^18",
+        "typescript": "^5.0.0"
+    },
+    "dependencies": {
+        "@pulumi/kubernetes": "^4.0.0",
+        "@pulumi/pulumi": "^3.113.0"
+    }
+}

03-k8s-infra/tsconfig.json

@@ -0,0 +1,18 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "outDir": "bin",
+        "target": "es2024",
+        "module": "nodenext",
+        "moduleResolution": "nodenext",
+        "sourceMap": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}

README.md

@@ -1,6 +1,6 @@
 # Homelab Infrastructure as Code
 
-A Pulumi-based IaC template for managing a Proxmox homelab. The goal is to replace manual GUI configuration and ad-hoc YAML stacks (LXC, VM, Docker, etc.) with version-controlled, reproducible infrastructure — starting with a highly available k3s cluster across multiple Proxmox nodes.
+A Pulumi-based IaC template for managing a Proxmox homelab. The goal is to replace manual GUI configuration and ad-hoc YAML stacks with version-controlled, reproducible infrastructure — a highly available k3s cluster across multiple Proxmox nodes, bootstrapped and configured end-to-end from code.
 
 This repo is intentionally abstract: credentials are never hardcoded, making it easy to fork and adapt as a template for your own homelab.
 
@@ -16,18 +16,30 @@ This repo is intentionally abstract: credentials are never hardcoded, making it
 
 ```
 .
-├── proxmox-infra/          # Pulumi TypeScript stack — VMs & LXC on Proxmox
-│   ├── index.ts            # All Pulumi resources
-│   ├── Pulumi.yaml         # Stack project definition
-│   └── Pulumi.dev.yaml     # Encrypted stack config (gitignored)
-├── .gitea/
-│   └── workflows/
-│       └── deploy-proxmox-infra.yaml   # Gitea Actions CI/CD pipeline
+├── proxmox-infra/          # Stack 1 — VMs & DHCP on Proxmox + pfSense
+│   ├── index.ts
+│   ├── Pulumi.yaml
+│   └── sdks/pfsense/       # Locally bundled @pulumi/pfsense SDK
+├── k8s-bootstrap/          # Stack 2 — Install k3s on the VMs over SSH
+│   ├── index.ts
+│   └── Pulumi.yaml
+├── k8s-infra/              # Stack 3 — Cluster-level infrastructure via Helm
+│   ├── index.ts
+│   └── Pulumi.yaml
+└── .gitea/workflows/       # Gitea Actions — one workflow per stack
 ```
 
-## Current stack: `proxmox-infra`
+## Stack overview
 
-Provisions a 5-node k3s cluster spread across two Proxmox hosts (`pve` and `pve-bckp`, third bare metal host to be added later for actual parity):
+The three stacks are deployed in order and linked via Pulumi StackReferences — outputs from one stack flow automatically into the next.
+
+```
+proxmox-infra  →  k8s-bootstrap  →  k8s-infra
+```
+
+### Stack 1: `proxmox-infra`
+
+Provisions a 5-node k3s cluster spread across two Proxmox hosts:
 
 | VM name      | Role   | Proxmox node |
 | ------------ | ------ | ------------ |
@@ -37,74 +49,151 @@ Provisions a 5-node k3s cluster spread across two Proxmox hosts (`pve` and `pve-
 | k3s-master-3 | master | pve-bckp     |
 | k3s-worker-2 | worker | pve-bckp     |
 
-Each node is a full clone of an Ubuntu Noble (24.04) cloud-image template, with cloud-init injecting hostname, user credentials, and SSH key at boot.
+Each VM is a full clone of an Ubuntu Noble (24.04) cloud-image template. Cloud-init injects the hostname, user credentials, and SSH keys at boot. An ED25519 SSH keypair is generated once and stored in Pulumi state — the public key goes into every VM, the private key is exported as a stack output for `k8s-bootstrap` to consume via StackReference.
 
-**Tech stack:**
+Each VM's MAC address is registered as a DHCPv4 static mapping in pfSense so nodes always receive their designated IPs.
 
-- [Pulumi](https://www.pulumi.com/) with TypeScript
-- [`@muhlba91/pulumi-proxmoxve`](https://github.com/muhlba91/pulumi-provider-proxmoxve) v8.x community provider
-- Self-hosted Pulumi state backend (PostgreSQL)
-- Gitea Actions for CI/CD
+**Providers:** `@muhlba91/pulumi-proxmoxve` v8.x, `@pulumi/pfsense` (locally bundled), `@pulumi/tls`
+
+**Exports:** `vmIds`, `ciRunnerPrivateKey`, `pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`, `master1Ip` … `worker2Ip`
+
+### Stack 2: `k8s-bootstrap`
+
+Installs k3s on the Proxmox VMs over SSH using `@pulumi/command`. Reads VM IDs, Proxmox credentials, and the SSH private key from `proxmox-infra` via StackReference — no manual key distribution needed.
+
+Deployment sequence:
+
+1. Start all 5 VMs via the Proxmox REST API
+2. Wait for port 22 to open on each node
+3. Install k3s on `k3s-master-1` with `--cluster-init` (embedded etcd)
+4. Join `k3s-master-2` and `k3s-master-3` as additional etcd nodes
+5. Join `k3s-worker-1` and `k3s-worker-2` as agent nodes
+6. Read `/etc/rancher/k3s/k3s.yaml` from master-1, patch the server URL, export as a secret stack output
+
+**Exports:** `kubeconfig` (secret)
+
+### Stack 3: `k8s-infra`
+
+Deploys cluster-level infrastructure via Helm and the `@pulumi/kubernetes` provider. Reads `kubeconfig` from `k8s-bootstrap` via StackReference.
+
+| Component            | What it does                                                               |
+| -------------------- | -------------------------------------------------------------------------- |
+| NFS CSI Driver       | `csi-driver-nfs` Helm chart (v4.12.0) — enables dynamic NFS-backed PVCs    |
+| TrueNAS StorageClass | `truenas-nfs` StorageClass backed by an NFS share on TrueNAS               |
+| cert-manager         | `cert-manager` Helm chart — certificate lifecycle management               |
+| ClusterIssuer        | `letsencrypt-prod` — DNS-01 via Cloudflare, issues Let's Encrypt TLS certs |
 
 ## Prerequisites
 
 - [Pulumi CLI](https://www.pulumi.com/docs/install/) installed
 - Node.js 18+ and npm
-- Access to a Proxmox node with an API token
+- Two Proxmox nodes with API tokens
+- pfSense with REST API credentials (used for DHCPv4 static mapping)
+- TrueNAS with an NFS share (no API key needed — CSI driver connects via NFS protocol directly)
+- A Cloudflare account with an API token scoped to DNS edit on your zone
 - A self-hosted Pulumi state backend (PostgreSQL connection string)
 - Gitea instance for CI/CD (optional for local use)
 
 ## Getting started
 
-### 1. Clone and install
+Deploy the stacks **in order**. Each stack must be fully deployed before the next one runs.
+
+### Stack 1 — `proxmox-infra`
 
 ```bash
-git clone <your-repo-url>
 cd proxmox-infra
-npm install
+npm install      # also compiles the bundled pfSense SDK
+pulumi stack init dev
 ```
 
-### 2. Configure credentials
-
-All secrets are stored as encrypted Pulumi config values — never in plain environment variables or committed files.
+Set secrets:
 
 ```bash
-# Set Proxmox API credentials
+# Proxmox API credentials
 pulumi config set --secret pve1Endpoint  https://<proxmox-host-1>:8006
 pulumi config set --secret pve1ApiToken  <user>@pam!<token-id>=<uuid>
 pulumi config set --secret pve2Endpoint  https://<proxmox-host-2>:8006
 pulumi config set --secret pve2ApiToken  <user>@pam!<token-id>=<uuid>
 
-# Set VM credentials
+# VM credentials
 pulumi config set --secret k3sVmPassword  <vm-password>
 pulumi config set --secret sshPvePublicKey "ssh-ed25519 AAAA..."
+
+# pfSense credentials (used for DHCPv4 static mappings)
+pulumi config set --secret pfSenseUrl      https://<pfsense-host>
+pulumi config set --secret pfSenseUser     <admin-username>
+pulumi config set --secret pfSensePassword <admin-password>
+
+# Static IP addresses assigned to each k3s node
+pulumi config set --secret master1Ip  <ip-for-k3s-master-1>
+pulumi config set --secret master2Ip  <ip-for-k3s-master-2>
+pulumi config set --secret worker1Ip  <ip-for-k3s-worker-1>
+pulumi config set --secret master3Ip  <ip-for-k3s-master-3>
+pulumi config set --secret worker2Ip  <ip-for-k3s-worker-2>
 ```
 
-Pulumi encrypts these values into `Pulumi.dev.yaml` using your `PULUMI_CONFIG_PASSPHRASE`.
-
-### 3. Set the state backend
+Deploy:
 
 ```bash
 export PULUMI_BACKEND_URL=postgresql://<user>:<pass>@<host>/<db>
 export PULUMI_CONFIG_PASSPHRASE=<your-passphrase>
+
+pulumi preview    # inspect before touching anything
+pulumi up --yes
 ```
 
-### 4. Preview and deploy
+### Stack 2 — `k8s-bootstrap`
 
 ```bash
-# See what will change before touching anything
-pulumi preview
+cd ../k8s-bootstrap
+npm install
+pulumi stack init dev
 
-# Sync Pulumi state with actual Proxmox state (run after any manual GUI changes)
-pulumi refresh --yes
+pulumi config set --secret k3sToken "$(openssl rand -hex 32)"
+```
 
-# Deploy
-pulumi refresh --yes && pulumi up --yes
+Node IPs and Proxmox credentials are read automatically from `proxmox-infra` outputs via StackReference — do not set them here.
+
+```bash
+pulumi up --yes
+```
+
+### Stack 3 — `k8s-infra`
+
+**TrueNAS one-time setup** (before deploying this stack):
+
+1. Create dataset `tank/k8s`
+2. Add an NFS share for that dataset
+
+    - Acl Type: Set this to POSIX (or generic Unix permissions) rather than SMB/NFSv4 ACLs if you are on TrueNAS SCALE, as Kubernetes handles basic Unix permissions natively.
+    - Maproot User / Maproot Group: Set this to root and root
+    - Allowed Hosts/Networks: Restrict this share specifically to the IP addresses or the CIDR block of your Proxmox K3s nodes (e.g., 192.168.1.50)
+
+3. In Network → Allowed Networks, permit your LAN subnet (e.g. `192.168.1.0/24`)
+
+```bash
+cd ../k8s-infra
+npm install
+pulumi stack init dev
+
+# kubeconfig from k8s-bootstrap
+KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets)
+pulumi config set --secret kubeconfig "$KUBECONFIG"
+
+# TrueNAS NFS
+pulumi config set --secret truenasHost    <truenas-ip>
+pulumi config set --secret truenasNfsPath /mnt/tank/k8s
+
+# cert-manager + Cloudflare DNS-01
+pulumi config set --secret cloudflareApiToken <cf-token>
+pulumi config set --secret letsencryptEmail   <your-email>
+
+pulumi up --yes
 ```
 
 ## CI/CD (Gitea Actions)
 
-The workflow at `.gitea/workflows/deploy-proxmox-infra.yaml` runs automatically:
+One workflow per stack under `.gitea/workflows/`. Each triggers on changes to its own stack directory.
 
 | Event                 | Action                         |
 | --------------------- | ------------------------------ |
@@ -114,34 +203,37 @@ The workflow at `.gitea/workflows/deploy-proxmox-infra.yaml` runs automatically:
 
 ### Required Gitea secrets
 
-Configure these under **Settings → Actions → Secrets** in your Gitea repo:
+Configure under **Settings → Actions → Secrets**:
 
-| Secret                     | Description                                        |
-| -------------------------- | -------------------------------------------------- |
-| `PULUMI_BACKEND_URL`       | PostgreSQL connection string for the state backend |
-| `PULUMI_CONFIG_PASSPHRASE` | Passphrase to decrypt secrets in `Pulumi.dev.yaml` |
-| `PULUMI_DEV_YAML`          | Base64-encoded content of `Pulumi.dev.yaml`        |
+| Secret                          | Used by                     | Description                                    |
+| ------------------------------- | --------------------------- | ---------------------------------------------- |
+| `PULUMI_BACKEND_URL`            | all workflows               | PostgreSQL connection string for state backend |
+| `PULUMI_CONFIG_PASSPHRASE`      | all workflows               | Passphrase to decrypt secrets                  |
+| `PROXMOX_INFRA_PULUMI_DEV_YAML` | `deploy-proxmox-infra.yaml` | Base64-encoded `proxmox-infra/Pulumi.dev.yaml` |
+| `K8S_BOOTSTRAP_PULUMI_DEV_YAML` | `deploy-k8s-bootstrap.yaml` | Base64-encoded `k8s-bootstrap/Pulumi.dev.yaml` |
+| `K8S_INFRA_PULUMI_DEV_YAML`     | `deploy-k8s-infra.yaml`     | Base64-encoded `k8s-infra/Pulumi.dev.yaml`     |
 
-`Pulumi.dev.yaml` is gitignored because it contains your encryption salt. Whenever it changes (e.g. after adding or rotating a secret), re-encode it and paste the output into the Gitea secret:
+`Pulumi.dev.yaml` files are gitignored (they contain your encryption salt). Whenever one changes, re-encode and paste into the corresponding Gitea secret:
 
 ```bash
 base64 -w 0 proxmox-infra/Pulumi.dev.yaml
+base64 -w 0 k8s-bootstrap/Pulumi.dev.yaml
+base64 -w 0 k8s-infra/Pulumi.dev.yaml
 ```
 
 ## Adapting this as a template
 
 1. Fork or copy the repo
-2. Update node names (`pve`, `pve-bckp`) and datastore IDs in `index.ts` to match your setup
+2. Update Proxmox node names (`pve`, `pve-bckp`) and datastore IDs in `proxmox-infra/index.ts`
 3. Add or remove VMs from the `nodeConfigs` array
 4. Set your own secrets with `pulumi config set --secret`
-5. Point the CI/CD workflow at your own Git instance
+5. Point the CI/CD workflows at your own Gitea instance
 
 ## Roadmap
 
 - LXC container management
+- `k8s-apps` stack — application deployments on the cluster
 - Docker / Compose stack provisioning
-- Network and firewall rules
-- Automated k3s bootstrapping (kubeconfig export)
-- Additional worker nodes and storage volumes
+
 - Migrate secrets management to [OpenBao](https://openbao.org/) — replace `PULUMI_CONFIG_PASSPHRASE` and manual `Pulumi.dev.yaml` encoding with a self-hosted vault
-- Add a third bare metal proxmox instance to create an actual 3 node parity.
+- Add a third bare metal Proxmox node for true 3-node HA parity