Merge pull request 'Bug/rework ci triggers' (#7) from bug/rework-ci-triggers into main

Reviewed-on: #7

.env.local

@@ -1,3 +0,0 @@
-GITEA_API_URL=https://gitea.kasuns.website/kasun/homelab-infrastructure-as-code.git
-GITEA_TOKEN=ba3fd0f4851aa627e2088da1f94a596646ba2de7
-

.gitea/workflows/01-deploy-proxmox-infra.yaml

@@ -0,0 +1,97 @@
+name: Deploy Proxmox Infra
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "01-proxmox-infra/**"
+      - ".gitea/workflows/**"
+
+jobs:
+  preview:
+    name: Pulumi Preview
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 01-proxmox-infra/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm ci
+        working-directory: 01-proxmox-infra
+
+      - name: Install Pulumi CLI
+        run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
+
+      - name: Generate Local pfSense SDK
+        run: pulumi package add terraform-provider marshallford/pfsense 0.22.0
+        working-directory: 01-proxmox-infra
+
+      - name: Preview
+        uses: pulumi/actions@v5
+        with:
+          command: preview
+          stack-name: dev
+          work-dir: 01-proxmox-infra
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+  deploy:
+    name: Pulumi Deploy
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 01-proxmox-infra/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm ci
+        working-directory: 01-proxmox-infra
+
+      - name: Install Pulumi CLI
+        run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
+
+      - name: Generate Local pfSense SDK
+        run: pulumi package add terraform-provider marshallford/pfsense 0.22.0
+        working-directory: 01-proxmox-infra
+
+      - name: Refresh State
+        run: |
+          pulumi login "$PULUMI_BACKEND_URL"
+          pulumi refresh --yes --stack dev \
+            --target 'urn:pulumi:dev::proxmox-infra::proxmoxve*' \
+            --target 'urn:pulumi:dev::proxmox-infra::tls*'
+        working-directory: 01-proxmox-infra
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+          PULUMI_BACKEND_URL: ${{ secrets.PULUMI_BACKEND_URL }}
+
+      - name: Deploy
+        uses: pulumi/actions@v5
+        with:
+          command: up
+          stack-name: dev
+          work-dir: 01-proxmox-infra
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

.gitea/workflows/02-deploy-k8s-bootstrap.yaml

@@ -0,0 +1,81 @@
+name: Deploy k8s Bootstrap
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "02-k8s-bootstrap/**"
+      - ".gitea/workflows/02-deploy-k8s-bootstrap.yaml"
+
+jobs:
+  preview:
+    name: Pulumi Preview
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > 02-k8s-bootstrap/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm install
+        working-directory: 02-k8s-bootstrap
+
+      - name: Preview
+        uses: pulumi/actions@v5
+        with:
+          command: preview
+          stack-name: dev
+          work-dir: 02-k8s-bootstrap
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+  deploy:
+    name: Bootstrap k3s Cluster
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > 02-k8s-bootstrap/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm install
+        working-directory: 02-k8s-bootstrap
+
+      - name: Refresh State
+        uses: pulumi/actions@v5
+        with:
+          command: refresh
+          stack-name: dev
+          work-dir: 02-k8s-bootstrap
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+      - name: Deploy
+        uses: pulumi/actions@v5
+        with:
+          command: up
+          stack-name: dev
+          work-dir: 02-k8s-bootstrap
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

.gitea/workflows/03-deploy-k8s-infra.yaml

@@ -1,4 +1,4 @@
-name: Deploy Proxmox Infra
+name: Deploy k8s Infra
 
 on:
   workflow_dispatch:
@@ -6,13 +6,13 @@ on:
     branches:
       - main
     paths:
-      - 'proxmox-infra/**'
+      - '03-k8s-infra/**'
       - '.gitea/workflows/**'
   pull_request:
     branches:
       - main
     paths:
-      - 'proxmox-infra/**'
+      - '03-k8s-infra/**'
       - '.gitea/workflows/**'
 
 jobs:
@@ -30,18 +30,21 @@ jobs:
           node-version: '24'
 
       - name: Restore Stack Config
-        run: echo "${{ secrets.PULUMI_DEV_YAML }}" | base64 -d > proxmox-infra/Pulumi.dev.yaml
+        run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 03-k8s-infra/Pulumi.dev.yaml
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
 
       - name: Install Dependencies
-        run: npm install
+        run: npm ci
-        working-directory: proxmox-infra
+        working-directory: 03-k8s-infra
 
       - name: Preview
         uses: pulumi/actions@v5
         with:
           command: preview
           stack-name: dev
-          work-dir: proxmox-infra
+          work-dir: 03-k8s-infra
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
@@ -60,18 +63,21 @@ jobs:
           node-version: '24'
 
       - name: Restore Stack Config
-        run: echo "${{ secrets.PULUMI_DEV_YAML }}" | base64 -d > proxmox-infra/Pulumi.dev.yaml
+        run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 03-k8s-infra/Pulumi.dev.yaml
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
 
       - name: Install Dependencies
-        run: npm install
+        run: npm ci
-        working-directory: proxmox-infra
+        working-directory: 03-k8s-infra
 
       - name: Refresh State
         uses: pulumi/actions@v5
         with:
           command: refresh
           stack-name: dev
-          work-dir: proxmox-infra
+          work-dir: 03-k8s-infra
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
@@ -81,7 +87,7 @@ jobs:
         with:
           command: up
           stack-name: dev
-          work-dir: proxmox-infra
+          work-dir: 03-k8s-infra
           cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
         env:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

.gitea/workflows/deploy-k8s-bootstrap.yaml

@@ -1,115 +0,0 @@
-name: Deploy k8s Bootstrap
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-    paths:
-      - 'k8s-bootstrap/**'
-      - '.gitea/workflows/deploy-k8s-bootstrap.yaml'
-  pull_request:
-    branches:
-      - main
-    paths:
-      - 'k8s-bootstrap/**'
-      - '.gitea/workflows/deploy-k8s-bootstrap.yaml'
-
-jobs:
-  preview:
-    name: Pulumi Preview
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '24'
-
-      - name: Restore Stack Config
-        run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
-
-      - name: Install Dependencies
-        run: npm install
-        working-directory: k8s-bootstrap
-
-      - name: Preview
-        uses: pulumi/actions@v5
-        with:
-          command: preview
-          stack-name: dev
-          work-dir: k8s-bootstrap
-          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
-        env:
-          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
-
-  deploy:
-    name: Bootstrap k3s Cluster
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '24'
-
-      - name: Restore Stack Config
-        run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
-
-      - name: Install Dependencies
-        run: npm install
-        working-directory: k8s-bootstrap
-
-      - name: Refresh State
-        uses: pulumi/actions@v5
-        with:
-          command: refresh
-          stack-name: dev
-          work-dir: k8s-bootstrap
-          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
-        env:
-          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
-
-      - name: Deploy
-        uses: pulumi/actions@v5
-        with:
-          command: up
-          stack-name: dev
-          work-dir: k8s-bootstrap
-          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
-        env:
-          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
-
-      # Propagate kubeconfig to the downstream stacks so their next deploy picks it up
-      - name: Propagate kubeconfig to k8s-infra
-        run: |
-          echo "${{ secrets.K8S_INFRA_DEV_YAML }}" | base64 -d > k8s-infra/Pulumi.dev.yaml
-          cd k8s-infra && npm install
-          KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \
-            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev)
-          pulumi config set --secret kubeconfig "$KUBECONFIG" \
-            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev
-          # Re-encode updated config for the secret (update manually in Gitea after first run)
-          base64 -w 0 Pulumi.dev.yaml
-        working-directory: .
-        env:
-          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
-
-      - name: Propagate kubeconfig to k8s-apps
-        run: |
-          echo "${{ secrets.K8S_APPS_DEV_YAML }}" | base64 -d > k8s-apps/Pulumi.dev.yaml
-          cd k8s-apps && npm install
-          KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \
-            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev)
-          pulumi config set --secret kubeconfig "$KUBECONFIG" \
-            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev
-          base64 -w 0 Pulumi.dev.yaml
-        working-directory: .
-        env:
-          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

.gitignore

@@ -3,4 +3,6 @@
 .vscode
 node_modules/
 bin/
-Pulumi.dev.yaml
+Pulumi.dev.yaml
+sdks/
+.env

01-proxmox-infra/Pulumi.yaml

@@ -8,3 +8,9 @@ config:
   pulumi:tags:
     value:
       pulumi:template: typescript
+packages:
+  pfsense:
+    source: terraform-provider
+    version: 1.1.3
+    parameters:
+      - marshallford/pfsense

01-proxmox-infra/index.ts

@@ -1,18 +1,29 @@
 import * as pulumi from "@pulumi/pulumi";
 import * as proxmox from "@muhlba91/pulumi-proxmoxve";
 import * as tls from "@pulumi/tls";
+import * as pfsense from "@pulumi/pfsense";
 
 const config = new pulumi.Config();
 
-// ---------------------------------------------------------------------------
-// Providers — one per standalone Proxmox machine
-// ---------------------------------------------------------------------------
-
 const pve1Endpoint = config.requireSecret("pve1Endpoint");
 const pve1ApiToken = config.requireSecret("pve1ApiToken");
 const pve2Endpoint = config.requireSecret("pve2Endpoint");
 const pve2ApiToken = config.requireSecret("pve2ApiToken");
 
+const pfSenseUrl = config.requireSecret("pfSenseUrl");
+const pfSenseUser = config.requireSecret("pfSenseUser");
+const pfSensePassword = config.requireSecret("pfSensePassword");
+
+const master1Ip = config.requireSecret("master1Ip");
+const master2Ip = config.requireSecret("master2Ip");
+const worker1Ip = config.requireSecret("worker1Ip");
+const master3Ip = config.requireSecret("master3Ip");
+const worker2Ip = config.requireSecret("worker2Ip");
+
+// ---------------------------------------------------------------------------
+// Providers — one per standalone Proxmox machine
+// ---------------------------------------------------------------------------
+
 const pveProvider = new proxmox.Provider("pve", {
   endpoint: pve1Endpoint,
   apiToken: pve1ApiToken,
@@ -25,6 +36,17 @@ const pveBckpProvider = new proxmox.Provider("pve-bckp", {
   insecure: true,
 });
 
+// ---------------------------------------------------------------------------
+// Providers — PfSense
+// ---------------------------------------------------------------------------
+
+const pfSenseProvider = new pfsense.Provider("pfsense", {
+  url: pfSenseUrl,
+  username: pfSenseUser,
+  password: pfSensePassword,
+  tlsSkipVerify: true,
+});
+
 // ---------------------------------------------------------------------------
 // CI runner SSH keypair — generated once, stored in Pulumi state backend.
 // Public key goes into every VM; private key is exported for k8s-bootstrap.
@@ -49,7 +71,7 @@ const ubuntuImagePve = new proxmox.download.File(
     contentType: "import",
     fileName: "noble-server-cloudimg-amd64.qcow2",
     url: ubuntuNobleUrl,
-    overwrite: true,
+    overwrite: false,
     overwriteUnmanaged: true,
   },
   { provider: pveProvider },
@@ -63,7 +85,7 @@ const ubuntuImagePveBckp = new proxmox.download.File(
     contentType: "import",
     fileName: "noble-server-cloudimg-amd64.qcow2",
     url: ubuntuNobleUrl,
-    overwrite: true,
+    overwrite: false,
     overwriteUnmanaged: true,
   },
   { provider: pveBckpProvider },
@@ -113,7 +135,7 @@ const pveTemplate = new proxmox.VmLegacy(
       },
     ],
   },
-  { provider: pveProvider },
+  { provider: pveProvider, ignoreChanges: ["disks"] },
 );
 
 const pveBckpTemplate = new proxmox.VmLegacy(
@@ -133,7 +155,7 @@ const pveBckpTemplate = new proxmox.VmLegacy(
       },
     ],
   },
-  { provider: pveBckpProvider },
+  { provider: pveBckpProvider, ignoreChanges: ["disks"] },
 );
 
 // ---------------------------------------------------------------------------
@@ -150,6 +172,7 @@ interface NodeConfig {
   provider: proxmox.Provider;
   template: proxmox.VmLegacy;
   diskDatastore: string;
+  ip: pulumi.Output<string>;
 }
 
 const nodeConfigs: NodeConfig[] = [
@@ -160,6 +183,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveProvider,
     template: pveTemplate,
     diskDatastore: "local-lvm",
+    ip: master1Ip,
   },
   {
     name: "k3s-master-2",
@@ -168,6 +192,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveProvider,
     template: pveTemplate,
     diskDatastore: "local-lvm",
+    ip: master2Ip,
   },
   {
     name: "k3s-worker-1",
@@ -176,6 +201,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveProvider,
     template: pveTemplate,
     diskDatastore: "local-lvm",
+    ip: worker1Ip,
   },
   {
     name: "k3s-master-3",
@@ -184,6 +210,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveBckpProvider,
     template: pveBckpTemplate,
     diskDatastore: "local",
+    ip: master3Ip,
   },
   {
     name: "k3s-worker-2",
@@ -192,6 +219,7 @@ const nodeConfigs: NodeConfig[] = [
     provider: pveBckpProvider,
     template: pveBckpTemplate,
     diskDatastore: "local",
+    ip: worker2Ip,
   },
 ];
 
@@ -235,9 +263,9 @@ const k3sVms = nodeConfigs.map(
             username: "ubuntu",
             password: k3sVmPassword,
             keys: [
-                  sshPvePublicKey.apply((k) => k.trim()),
+              sshPvePublicKey.apply((k) => k.trim()),
-                  ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
+              ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
-                ],
+            ],
           },
         },
         networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
@@ -251,17 +279,27 @@ const k3sVms = nodeConfigs.map(
       {
         provider: node.provider,
         retainOnDelete: true,
-        ignoreChanges: ["clone"],
+        ignoreChanges: ["clone", "started", "disks"],
       },
     ),
 );
 
-export const clusterInfo = k3sVms.map((vm, index) => ({
+k3sVms.forEach((vmResource, i) => {
-  nodeName: vm.nodeName,
+  const assignedMac = vmResource.networkDevices.apply(
-  vmId: vm.vmId,
+    (nic) => nic[0].macAddress,
-  name: nodeConfigs[index].name,
+  );
-  role: nodeConfigs[index].role,
+
-}));
+  return new pfsense.Dhcpv4Staticmapping(
+    `${nodeConfigs[i].name}-dhcp`,
+    {
+      interface: "lan",
+      macAddress: assignedMac,
+      ipAddress: nodeConfigs[i].ip,
+      hostname: nodeConfigs[i].name,
+    },
+    { dependsOn: vmResource, provider: pfSenseProvider },
+  );
+});
 
 // Individual vmId exports — used by k8s-bootstrap to start VMs.
 // Order matches nodeConfigs: master-1, master-2, worker-1, master-3, worker-2.
@@ -278,3 +316,6 @@ export const ciRunnerPrivateKey = pulumi.secret(ciRunnerKey.privateKeyOpenssh);
 
 // Proxmox API credentials — consumed by k8s-bootstrap via StackReference.
 export { pve1Endpoint, pve1ApiToken, pve2Endpoint, pve2ApiToken };
+
+//k3s instance ips consumed by k8s-bootstrap.
+export { master1Ip, master2Ip, worker1Ip, master3Ip, worker2Ip };

01-proxmox-infra/package-lock.json

@@ -7,6 +7,7 @@
             "name": "proxmox-infra",
             "dependencies": {
                 "@muhlba91/pulumi-proxmoxve": "^8.2.1",
+                "@pulumi/pfsense": "file:sdks/pfsense",
                 "@pulumi/pulumi": "^3.113.0",
                 "@pulumi/tls": "^5.5.0"
             },
@@ -703,6 +704,10 @@
             "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==",
             "license": "BSD-3-Clause"
         },
+        "node_modules/@pulumi/pfsense": {
+            "resolved": "sdks/pfsense",
+            "link": true
+        },
         "node_modules/@pulumi/pulumi": {
             "version": "3.243.0",
             "resolved": "https://registry.npmjs.org/@pulumi/pulumi/-/pulumi-3.243.0.tgz",
@@ -2674,6 +2679,44 @@
             "engines": {
                 "node": ">=12"
             }
+        },
+        "sdks/pfsense": {
+            "name": "@pulumi/pfsense",
+            "version": "0.22.0",
+            "hasInstallScript": true,
+            "dependencies": {
+                "@pulumi/pulumi": "^3.238.0",
+                "@types/node": "^20",
+                "typescript": "^4.7.0"
+            }
+        },
+        "sdks/pfsense/node_modules/@types/node": {
+            "version": "20.19.41",
+            "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.41.tgz",
+            "integrity": "sha512-ECymXOukMnOoVkC2bb1Vc/w/836DXncOg5m8Xj1RH7xSHZJWNYY6Zh7EH477vcnD5egKNNfy2RpNOmuChhFPgQ==",
+            "license": "MIT",
+            "dependencies": {
+                "undici-types": "~6.21.0"
+            }
+        },
+        "sdks/pfsense/node_modules/typescript": {
+            "version": "4.9.5",
+            "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
+            "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+            "license": "Apache-2.0",
+            "bin": {
+                "tsc": "bin/tsc",
+                "tsserver": "bin/tsserver"
+            },
+            "engines": {
+                "node": ">=4.2.0"
+            }
+        },
+        "sdks/pfsense/node_modules/undici-types": {
+            "version": "6.21.0",
+            "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
+            "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+            "license": "MIT"
         }
     }
 }

01-proxmox-infra/package.json

@@ -7,7 +7,12 @@
     },
     "dependencies": {
         "@muhlba91/pulumi-proxmoxve": "^8.2.1",
+        "@pulumi/pfsense": "file:sdks/pfsense",
         "@pulumi/pulumi": "^3.113.0",
         "@pulumi/tls": "^5.5.0"
+    },
+    "imports": {
+    "#pfsense": "./sdks/pfsense/index.js",
+    "#pfsense/*": "./sdks/pfsense/*"
     }
 }

02-k8s-bootstrap/CLAUDE.md

@@ -5,7 +5,7 @@ Bootstraps a k3s cluster on the 5 Proxmox VMs created by `proxmox-infra`. Starts
 ## How it works
 
 1. Starts all 5 VMs via `POST /api2/json/nodes/{node}/qemu/{vmid}/status/start`
-2. Waits for port 22 to open on each VM (`nc -z`)
+2. Waits for port 22 to open on each VM (bash `/dev/tcp`)
 3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
 4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
 5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes
@@ -18,12 +18,6 @@ VM IDs and the CI runner SSH private key are read automatically from the `proxmo
 Run in this directory after `pulumi stack init dev`:
 
 ```bash
-# Same as proxmox-infra
-pulumi config set --secret pve1Endpoint "https://192.168.1.x:8006"
-pulumi config set --secret pve1ApiToken "root@pam!pulumi=<token>"
-pulumi config set --secret pve2Endpoint "https://192.168.1.y:8006"
-pulumi config set --secret pve2ApiToken "root@pam!pulumi=<token>"
-
 # Pre-shared k3s token — any strong random string
 pulumi config set --secret k3sToken "$(openssl rand -hex 32)"
 
@@ -35,6 +29,14 @@ pulumi config set worker1Ip "192.168.1.x"
 pulumi config set worker2Ip "192.168.1.x"
 ```
 
+Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`) are read automatically from the `proxmox-infra` stack outputs via StackReference — do **not** set them here.
+
+## Deployment order
+
+`proxmox-infra` must be deployed **before** k8s-bootstrap. The proxmox-infra stack exports the Proxmox credentials and VM IDs that k8s-bootstrap reads via StackReference. If proxmox-infra hasn't been re-deployed after its latest code changes, those outputs won't exist and k8s-bootstrap will fail.
+
+In CI, both workflows run as previews on PRs (no deploy). Trigger `workflow_dispatch` on `deploy-proxmox-infra` first, then run k8s-bootstrap.
+
 After setting config, re-encode `Pulumi.dev.yaml` and update the Gitea secret `K8S_BOOTSTRAP_DEV_YAML`:
 ```bash
 base64 -w 0 Pulumi.dev.yaml

02-k8s-bootstrap/index.ts

@@ -0,0 +1,235 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as command from "@pulumi/command";
+
+const config = new pulumi.Config();
+
+//fetch credentials from proxmox-infra
+const infraRef = new pulumi.StackReference(
+  `${pulumi.getOrganization()}/proxmox-infra/dev`,
+);
+
+// Proxmox API credentials — same as proxmox-infra stack
+const pve1Endpoint = infraRef.requireOutput(
+  "pve1Endpoint",
+) as pulumi.Output<string>;
+const pve1ApiToken = infraRef.requireOutput(
+  "pve1ApiToken",
+) as pulumi.Output<string>;
+const pve2Endpoint = infraRef.requireOutput(
+  "pve2Endpoint",
+) as pulumi.Output<string>;
+const pve2ApiToken = infraRef.requireOutput(
+  "pve2ApiToken",
+) as pulumi.Output<string>;
+
+// Node IPs — static DHCP leases set in the router
+const master1Ip = infraRef.requireOutput("master1Ip");
+const master2Ip = infraRef.requireOutput("master2Ip");
+const master3Ip = infraRef.requireOutput("master3Ip");
+const worker1Ip = infraRef.requireOutput("worker1Ip");
+const worker2Ip = infraRef.requireOutput("worker2Ip");
+
+// Pre-shared k3s cluster token
+const k3sToken = config.requireSecret("k3sToken");
+
+// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
+const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<
+  Record<string, number>
+>;
+const ciRunnerPrivateKey = infraRef.requireOutput(
+  "ciRunnerPrivateKey",
+) as pulumi.Output<string>;
+
+const master1VmId = vmIdsOutput.apply((ids) => String(ids.master1));
+const master2VmId = vmIdsOutput.apply((ids) => String(ids.master2));
+const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3));
+const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1));
+const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2));
+
+// SSH connection helper
+function conn(
+  ip: pulumi.Input<string>,
+): command.types.input.remote.ConnectionArgs {
+  return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
+}
+
+// ---------------------------------------------------------------------------
+// Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra)
+// triggers: vmId — forces re-run when VMs are deleted and recreated
+// ---------------------------------------------------------------------------
+
+const startMaster1 = new command.local.Command("start-master-1", {
+  create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`,
+  triggers: [master1VmId],
+  interpreter: ["/bin/bash", "-c"],
+});
+const startMaster2 = new command.local.Command("start-master-2", {
+  create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`,
+  triggers: [master2VmId],
+  interpreter: ["/bin/bash", "-c"],
+});
+const startWorker1 = new command.local.Command("start-worker-1", {
+  create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`,
+  triggers: [worker1VmId],
+  interpreter: ["/bin/bash", "-c"],
+});
+const startMaster3 = new command.local.Command("start-master-3", {
+  create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`,
+  triggers: [master3VmId],
+  interpreter: ["/bin/bash", "-c"],
+});
+const startWorker2 = new command.local.Command("start-worker-2", {
+  create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`,
+  triggers: [worker2VmId],
+  interpreter: ["/bin/bash", "-c"],
+});
+
+const allStarts = [
+  startMaster1,
+  startMaster2,
+  startMaster3,
+  startWorker1,
+  startWorker2,
+];
+
+// ---------------------------------------------------------------------------
+// Step 2: Wait for SSH on master-1, install k3s
+// ---------------------------------------------------------------------------
+
+const waitMaster1Ssh = new command.local.Command(
+  "wait-ssh-master-1",
+  {
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    triggers: [master1VmId],
+    interpreter: ["/bin/bash", "-c"],
+  },
+  { dependsOn: allStarts },
+);
+
+const installMaster1 = new command.remote.Command(
+  "install-k3s-master-1",
+  {
+    connection: conn(master1Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`,
+    triggers: [master1VmId],
+  },
+  { dependsOn: [waitMaster1Ssh] },
+);
+
+const waitK3sMaster1Ready = new command.remote.Command(
+  "wait-k3s-master-1-ready",
+  {
+    connection: conn(master1Ip),
+    create: `for i in $(seq 1 60); do sudo kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`,
+    triggers: [master1VmId],
+  },
+  { dependsOn: [installMaster1] },
+);
+
+// ---------------------------------------------------------------------------
+// Step 3: Wait for SSH on remaining masters, join cluster
+// ---------------------------------------------------------------------------
+
+const waitMaster2Ssh = new command.local.Command(
+  "wait-ssh-master-2",
+  {
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    triggers: [master2VmId],
+    interpreter: ["/bin/bash", "-c"],
+  },
+  { dependsOn: [waitK3sMaster1Ready] },
+);
+
+const waitMaster3Ssh = new command.local.Command(
+  "wait-ssh-master-3",
+  {
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    triggers: [master3VmId],
+    interpreter: ["/bin/bash", "-c"],
+  },
+  { dependsOn: [waitK3sMaster1Ready] },
+);
+
+const joinMaster2 = new command.remote.Command(
+  "join-k3s-master-2",
+  {
+    connection: conn(master2Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`,
+    triggers: [master2VmId],
+  },
+  { dependsOn: [waitMaster2Ssh] },
+);
+
+const joinMaster3 = new command.remote.Command(
+  "join-k3s-master-3",
+  {
+    connection: conn(master3Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`,
+    triggers: [master3VmId],
+  },
+  { dependsOn: [waitMaster3Ssh, joinMaster2] },
+);
+
+// ---------------------------------------------------------------------------
+// Step 4: Join worker nodes
+// ---------------------------------------------------------------------------
+
+const waitWorker1Ssh = new command.local.Command(
+  "wait-ssh-worker-1",
+  {
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    triggers: [worker1VmId],
+    interpreter: ["/bin/bash", "-c"],
+  },
+  { dependsOn: [joinMaster3] },
+);
+
+const waitWorker2Ssh = new command.local.Command(
+  "wait-ssh-worker-2",
+  {
+    create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
+    triggers: [worker2VmId],
+    interpreter: ["/bin/bash", "-c"],
+  },
+  { dependsOn: [joinMaster3] },
+);
+
+const joinWorker1 = new command.remote.Command(
+  "join-k3s-worker-1",
+  {
+    connection: conn(worker1Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
+    triggers: [worker1VmId],
+  },
+  { dependsOn: [waitWorker1Ssh] },
+);
+
+const joinWorker2 = new command.remote.Command(
+  "join-k3s-worker-2",
+  {
+    connection: conn(worker2Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
+    triggers: [worker2VmId],
+  },
+  { dependsOn: [waitWorker2Ssh] },
+);
+
+// ---------------------------------------------------------------------------
+// Step 5: Read kubeconfig from master-1, patch server URL for external access
+// ---------------------------------------------------------------------------
+
+const getKubeconfig = new command.remote.Command(
+  "get-kubeconfig",
+  {
+    connection: conn(master1Ip),
+    create: `sudo cat /etc/rancher/k3s/k3s.yaml`,
+    triggers: [master1VmId],
+  },
+  { dependsOn: [joinWorker1, joinWorker2] },
+);
+
+export const kubeconfig = pulumi.secret(
+  pulumi
+    .all([getKubeconfig.stdout, master1Ip])
+    .apply(([kc, ip]) => kc.replaceAll("127.0.0.1", ip as string).trim()),
+);

02-k8s-bootstrap/tsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "strict": true,
         "outDir": "bin",
-        "target": "es2020",
+        "target": "es2024",
         "module": "nodenext",
         "moduleResolution": "nodenext",
         "sourceMap": true,

03-k8s-infra/Pulumi.yaml

@@ -0,0 +1,6 @@
+name: k8s-infra
+description: Cluster-level infrastructure for the k3s homelab cluster
+runtime:
+  name: nodejs
+  options:
+    packagemanager: npm

03-k8s-infra/index.ts

@@ -0,0 +1,118 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as k8s from "@pulumi/kubernetes";
+
+const config = new pulumi.Config();
+
+//fetch credentials from k8s-bootstrap
+const infraRef = new pulumi.StackReference(
+  `${pulumi.getOrganization()}/k8s-bootstrap/dev`,
+);
+
+const kubeconfig = infraRef.requireOutput("kubeconfig");
+const truenasHost = config.requireSecret("truenasHost");
+const truenasNfsPath = config.requireSecret("truenasNfsPath");
+const cloudflareToken = config.requireSecret("cloudflareApiToken");
+const letsencryptEmail = config.requireSecret("letsencryptEmail");
+
+const k8sProvider = new k8s.Provider("k3s", { kubeconfig });
+const opts = (extras?: pulumi.ResourceOptions): pulumi.ResourceOptions => ({
+  provider: k8sProvider,
+  ...extras,
+});
+
+// ── 1. NFS CSI Driver ────────────────────────────────────────────────────────
+
+const nfsCsiDriver = new k8s.helm.v3.Release(
+  "nfs-csi-driver",
+  {
+    name: "csi-driver-nfs",
+    chart: "csi-driver-nfs",
+    repositoryOpts: {
+      repo: "https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts",
+    },
+    namespace: "kube-system",
+    version: "4.12.0",
+    values: {
+      kubeletDir: "/var/lib/kubelet",
+    },
+  },
+  opts(),
+);
+
+new k8s.storage.v1.StorageClass(
+  "truenas-nfs",
+  {
+    metadata: { name: "truenas-nfs" },
+    provisioner: "nfs.csi.k8s.io",
+    parameters: {
+      server: truenasHost,
+      share: truenasNfsPath,
+      mountPermissions: "0",
+    },
+    reclaimPolicy: "Retain",
+    volumeBindingMode: "Immediate",
+    allowVolumeExpansion: true,
+    mountOptions: ["nfsvers=4.1"],
+  },
+  opts({ dependsOn: [nfsCsiDriver] }),
+);
+
+// ── 2. cert-manager ──────────────────────────────────────────────────────────
+
+const certManager = new k8s.helm.v3.Release(
+  "cert-manager",
+  {
+    name: "cert-manager",
+    chart: "cert-manager",
+    repositoryOpts: { repo: "https://charts.jetstack.io" },
+    namespace: "cert-manager",
+    createNamespace: true,
+    timeout: 600,
+    values: {
+      crds: { enabled: true },
+    },
+  },
+  opts(),
+);
+
+// ── 3. Cloudflare token secret + ClusterIssuer ───────────────────────────────
+
+const cfSecret = new k8s.core.v1.Secret(
+  "cloudflare-token",
+  {
+    metadata: { name: "cloudflare-api-token", namespace: "cert-manager" },
+    stringData: { "api-token": cloudflareToken },
+  },
+  opts({ dependsOn: [certManager] }),
+);
+
+new k8s.apiextensions.CustomResource(
+  "letsencrypt-prod",
+  {
+    apiVersion: "cert-manager.io/v1",
+    kind: "ClusterIssuer",
+    metadata: { name: "letsencrypt-prod" },
+    spec: {
+      acme: {
+        server: "https://acme-v02.api.letsencrypt.org/directory",
+        email: letsencryptEmail,
+        privateKeySecretRef: { name: "letsencrypt-prod-account-key" },
+        solvers: [
+          {
+            dns01: {
+              cloudflare: {
+                apiTokenSecretRef: {
+                  name: cfSecret.metadata.name,
+                  key: "api-token",
+                },
+              },
+            },
+          },
+        ],
+      },
+    },
+  },
+  opts({ dependsOn: [certManager, cfSecret] }),
+);
+
+export const storageClass = "truenas-nfs";

03-k8s-infra/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "k8s-infra",
+    "main": "index.ts",
+    "devDependencies": {
+        "@types/node": "^18",
+        "typescript": "^5.0.0"
+    },
+    "dependencies": {
+        "@pulumi/kubernetes": "^4.0.0",
+        "@pulumi/pulumi": "^3.113.0"
+    }
+}

03-k8s-infra/tsconfig.json

@@ -0,0 +1,18 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "outDir": "bin",
+        "target": "es2024",
+        "module": "nodenext",
+        "moduleResolution": "nodenext",
+        "sourceMap": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}

README.md

@@ -1,6 +1,6 @@
 # Homelab Infrastructure as Code
 
-A Pulumi-based IaC template for managing a Proxmox homelab. The goal is to replace manual GUI configuration and ad-hoc YAML stacks (LXC, VM, Docker, etc.) with version-controlled, reproducible infrastructure — starting with a highly available k3s cluster across multiple Proxmox nodes.
+A Pulumi-based IaC template for managing a Proxmox homelab. The goal is to replace manual GUI configuration and ad-hoc YAML stacks with version-controlled, reproducible infrastructure — a highly available k3s cluster across multiple Proxmox nodes, bootstrapped and configured end-to-end from code.
 
 This repo is intentionally abstract: credentials are never hardcoded, making it easy to fork and adapt as a template for your own homelab.
 
@@ -16,18 +16,30 @@ This repo is intentionally abstract: credentials are never hardcoded, making it
 
 ```
 .
-├── proxmox-infra/          # Pulumi TypeScript stack — VMs & LXC on Proxmox
+├── proxmox-infra/          # Stack 1 — VMs & DHCP on Proxmox + pfSense
-│   ├── index.ts            # All Pulumi resources
+│   ├── index.ts
-│   ├── Pulumi.yaml         # Stack project definition
+│   ├── Pulumi.yaml
-│   └── Pulumi.dev.yaml     # Encrypted stack config (gitignored)
+│   └── sdks/pfsense/       # Locally bundled @pulumi/pfsense SDK
-├── .gitea/
+├── k8s-bootstrap/          # Stack 2 — Install k3s on the VMs over SSH
-│   └── workflows/
+│   ├── index.ts
-│       └── deploy-proxmox-infra.yaml   # Gitea Actions CI/CD pipeline
+│   └── Pulumi.yaml
+├── k8s-infra/              # Stack 3 — Cluster-level infrastructure via Helm
+│   ├── index.ts
+│   └── Pulumi.yaml
+└── .gitea/workflows/       # Gitea Actions — one workflow per stack
 ```
 
-## Current stack: `proxmox-infra`
+## Stack overview
 
-Provisions a 5-node k3s cluster spread across two Proxmox hosts (`pve` and `pve-bckp`, third bare metal host to be added later for actual parity):
+The three stacks are deployed in order and linked via Pulumi StackReferences — outputs from one stack flow automatically into the next.
+
+```
+proxmox-infra  →  k8s-bootstrap  →  k8s-infra
+```
+
+### Stack 1: `proxmox-infra`
+
+Provisions a 5-node k3s cluster spread across two Proxmox hosts:
 
 | VM name      | Role   | Proxmox node |
 | ------------ | ------ | ------------ |
@@ -37,74 +49,151 @@ Provisions a 5-node k3s cluster spread across two Proxmox hosts (`pve` and `pve-
 | k3s-master-3 | master | pve-bckp     |
 | k3s-worker-2 | worker | pve-bckp     |
 
-Each node is a full clone of an Ubuntu Noble (24.04) cloud-image template, with cloud-init injecting hostname, user credentials, and SSH key at boot.
+Each VM is a full clone of an Ubuntu Noble (24.04) cloud-image template. Cloud-init injects the hostname, user credentials, and SSH keys at boot. An ED25519 SSH keypair is generated once and stored in Pulumi state — the public key goes into every VM, the private key is exported as a stack output for `k8s-bootstrap` to consume via StackReference.
 
-**Tech stack:**
+Each VM's MAC address is registered as a DHCPv4 static mapping in pfSense so nodes always receive their designated IPs.
 
-- [Pulumi](https://www.pulumi.com/) with TypeScript
+**Providers:** `@muhlba91/pulumi-proxmoxve` v8.x, `@pulumi/pfsense` (locally bundled), `@pulumi/tls`
-- [`@muhlba91/pulumi-proxmoxve`](https://github.com/muhlba91/pulumi-provider-proxmoxve) v8.x community provider
+
-- Self-hosted Pulumi state backend (PostgreSQL)
+**Exports:** `vmIds`, `ciRunnerPrivateKey`, `pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`, `master1Ip` … `worker2Ip`
-- Gitea Actions for CI/CD
+
+### Stack 2: `k8s-bootstrap`
+
+Installs k3s on the Proxmox VMs over SSH using `@pulumi/command`. Reads VM IDs, Proxmox credentials, and the SSH private key from `proxmox-infra` via StackReference — no manual key distribution needed.
+
+Deployment sequence:
+
+1. Start all 5 VMs via the Proxmox REST API
+2. Wait for port 22 to open on each node
+3. Install k3s on `k3s-master-1` with `--cluster-init` (embedded etcd)
+4. Join `k3s-master-2` and `k3s-master-3` as additional etcd nodes
+5. Join `k3s-worker-1` and `k3s-worker-2` as agent nodes
+6. Read `/etc/rancher/k3s/k3s.yaml` from master-1, patch the server URL, export as a secret stack output
+
+**Exports:** `kubeconfig` (secret)
+
+### Stack 3: `k8s-infra`
+
+Deploys cluster-level infrastructure via Helm and the `@pulumi/kubernetes` provider. Reads `kubeconfig` from `k8s-bootstrap` via StackReference.
+
+| Component            | What it does                                                               |
+| -------------------- | -------------------------------------------------------------------------- |
+| NFS CSI Driver       | `csi-driver-nfs` Helm chart (v4.12.0) — enables dynamic NFS-backed PVCs    |
+| TrueNAS StorageClass | `truenas-nfs` StorageClass backed by an NFS share on TrueNAS               |
+| cert-manager         | `cert-manager` Helm chart — certificate lifecycle management               |
+| ClusterIssuer        | `letsencrypt-prod` — DNS-01 via Cloudflare, issues Let's Encrypt TLS certs |
 
 ## Prerequisites
 
 - [Pulumi CLI](https://www.pulumi.com/docs/install/) installed
 - Node.js 18+ and npm
-- Access to a Proxmox node with an API token
+- Two Proxmox nodes with API tokens
+- pfSense with REST API credentials (used for DHCPv4 static mapping)
+- TrueNAS with an NFS share (no API key needed — CSI driver connects via NFS protocol directly)
+- A Cloudflare account with an API token scoped to DNS edit on your zone
 - A self-hosted Pulumi state backend (PostgreSQL connection string)
 - Gitea instance for CI/CD (optional for local use)
 
 ## Getting started
 
-### 1. Clone and install
+Deploy the stacks **in order**. Each stack must be fully deployed before the next one runs.
+
+### Stack 1 — `proxmox-infra`
 
 ```bash
-git clone <your-repo-url>
 cd proxmox-infra
-npm install
+npm install      # also compiles the bundled pfSense SDK
+pulumi stack init dev
 ```
 
-### 2. Configure credentials
+Set secrets:
-
-All secrets are stored as encrypted Pulumi config values — never in plain environment variables or committed files.
 
 ```bash
-# Set Proxmox API credentials
+# Proxmox API credentials
 pulumi config set --secret pve1Endpoint  https://<proxmox-host-1>:8006
 pulumi config set --secret pve1ApiToken  <user>@pam!<token-id>=<uuid>
 pulumi config set --secret pve2Endpoint  https://<proxmox-host-2>:8006
 pulumi config set --secret pve2ApiToken  <user>@pam!<token-id>=<uuid>
 
-# Set VM credentials
+# VM credentials
 pulumi config set --secret k3sVmPassword  <vm-password>
 pulumi config set --secret sshPvePublicKey "ssh-ed25519 AAAA..."
+
+# pfSense credentials (used for DHCPv4 static mappings)
+pulumi config set --secret pfSenseUrl      https://<pfsense-host>
+pulumi config set --secret pfSenseUser     <admin-username>
+pulumi config set --secret pfSensePassword <admin-password>
+
+# Static IP addresses assigned to each k3s node
+pulumi config set --secret master1Ip  <ip-for-k3s-master-1>
+pulumi config set --secret master2Ip  <ip-for-k3s-master-2>
+pulumi config set --secret worker1Ip  <ip-for-k3s-worker-1>
+pulumi config set --secret master3Ip  <ip-for-k3s-master-3>
+pulumi config set --secret worker2Ip  <ip-for-k3s-worker-2>
 ```
 
-Pulumi encrypts these values into `Pulumi.dev.yaml` using your `PULUMI_CONFIG_PASSPHRASE`.
+Deploy:
-
-### 3. Set the state backend
 
 ```bash
 export PULUMI_BACKEND_URL=postgresql://<user>:<pass>@<host>/<db>
 export PULUMI_CONFIG_PASSPHRASE=<your-passphrase>
+
+pulumi preview    # inspect before touching anything
+pulumi up --yes
 ```
 
-### 4. Preview and deploy
+### Stack 2 — `k8s-bootstrap`
 
 ```bash
-# See what will change before touching anything
+cd ../k8s-bootstrap
-pulumi preview
+npm install
+pulumi stack init dev
 
-# Sync Pulumi state with actual Proxmox state (run after any manual GUI changes)
+pulumi config set --secret k3sToken "$(openssl rand -hex 32)"
-pulumi refresh --yes
+```
 
-# Deploy
+Node IPs and Proxmox credentials are read automatically from `proxmox-infra` outputs via StackReference — do not set them here.
-pulumi refresh --yes && pulumi up --yes
+
+```bash
+pulumi up --yes
+```
+
+### Stack 3 — `k8s-infra`
+
+**TrueNAS one-time setup** (before deploying this stack):
+
+1. Create dataset `tank/k8s`
+2. Add an NFS share for that dataset
+
+    - Acl Type: Set this to POSIX (or generic Unix permissions) rather than SMB/NFSv4 ACLs if you are on TrueNAS SCALE, as Kubernetes handles basic Unix permissions natively.
+    - Maproot User / Maproot Group: Set this to root and root
+    - Allowed Hosts/Networks: Restrict this share specifically to the IP addresses or the CIDR block of your Proxmox K3s nodes (e.g., 192.168.1.50)
+
+3. In Network → Allowed Networks, permit your LAN subnet (e.g. `192.168.1.0/24`)
+
+```bash
+cd ../k8s-infra
+npm install
+pulumi stack init dev
+
+# kubeconfig from k8s-bootstrap
+KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets)
+pulumi config set --secret kubeconfig "$KUBECONFIG"
+
+# TrueNAS NFS
+pulumi config set --secret truenasHost    <truenas-ip>
+pulumi config set --secret truenasNfsPath /mnt/tank/k8s
+
+# cert-manager + Cloudflare DNS-01
+pulumi config set --secret cloudflareApiToken <cf-token>
+pulumi config set --secret letsencryptEmail   <your-email>
+
+pulumi up --yes
 ```
 
 ## CI/CD (Gitea Actions)
 
-The workflow at `.gitea/workflows/deploy-proxmox-infra.yaml` runs automatically:
+One workflow per stack under `.gitea/workflows/`. Each triggers on changes to its own stack directory.
 
 | Event                 | Action                         |
 | --------------------- | ------------------------------ |
@@ -114,34 +203,37 @@ The workflow at `.gitea/workflows/deploy-proxmox-infra.yaml` runs automatically:
 
 ### Required Gitea secrets
 
-Configure these under **Settings → Actions → Secrets** in your Gitea repo:
+Configure under **Settings → Actions → Secrets**:
 
-| Secret                     | Description                                        |
+| Secret                          | Used by                     | Description                                    |
-| -------------------------- | -------------------------------------------------- |
+| ------------------------------- | --------------------------- | ---------------------------------------------- |
-| `PULUMI_BACKEND_URL`       | PostgreSQL connection string for the state backend |
+| `PULUMI_BACKEND_URL`            | all workflows               | PostgreSQL connection string for state backend |
-| `PULUMI_CONFIG_PASSPHRASE` | Passphrase to decrypt secrets in `Pulumi.dev.yaml` |
+| `PULUMI_CONFIG_PASSPHRASE`      | all workflows               | Passphrase to decrypt secrets                  |
-| `PULUMI_DEV_YAML`          | Base64-encoded content of `Pulumi.dev.yaml`        |
+| `PROXMOX_INFRA_PULUMI_DEV_YAML` | `deploy-proxmox-infra.yaml` | Base64-encoded `proxmox-infra/Pulumi.dev.yaml` |
+| `K8S_BOOTSTRAP_PULUMI_DEV_YAML` | `deploy-k8s-bootstrap.yaml` | Base64-encoded `k8s-bootstrap/Pulumi.dev.yaml` |
+| `K8S_INFRA_PULUMI_DEV_YAML`     | `deploy-k8s-infra.yaml`     | Base64-encoded `k8s-infra/Pulumi.dev.yaml`     |
 
-`Pulumi.dev.yaml` is gitignored because it contains your encryption salt. Whenever it changes (e.g. after adding or rotating a secret), re-encode it and paste the output into the Gitea secret:
+`Pulumi.dev.yaml` files are gitignored (they contain your encryption salt). Whenever one changes, re-encode and paste into the corresponding Gitea secret:
 
 ```bash
 base64 -w 0 proxmox-infra/Pulumi.dev.yaml
+base64 -w 0 k8s-bootstrap/Pulumi.dev.yaml
+base64 -w 0 k8s-infra/Pulumi.dev.yaml
 ```
 
 ## Adapting this as a template
 
 1. Fork or copy the repo
-2. Update node names (`pve`, `pve-bckp`) and datastore IDs in `index.ts` to match your setup
+2. Update Proxmox node names (`pve`, `pve-bckp`) and datastore IDs in `proxmox-infra/index.ts`
 3. Add or remove VMs from the `nodeConfigs` array
 4. Set your own secrets with `pulumi config set --secret`
-5. Point the CI/CD workflow at your own Git instance
+5. Point the CI/CD workflows at your own Gitea instance
 
 ## Roadmap
 
 - LXC container management
+- `k8s-apps` stack — application deployments on the cluster
 - Docker / Compose stack provisioning
-- Network and firewall rules
+
-- Automated k3s bootstrapping (kubeconfig export)
-- Additional worker nodes and storage volumes
 - Migrate secrets management to [OpenBao](https://openbao.org/) — replace `PULUMI_CONFIG_PASSPHRASE` and manual `Pulumi.dev.yaml` encoding with a self-hosted vault
-- Add a third bare metal proxmox instance to create an actual 3 node parity.
+- Add a third bare metal Proxmox node for true 3-node HA parity

k8s-bootstrap/index.ts

@@ -1,155 +0,0 @@
-import * as pulumi from "@pulumi/pulumi";
-import * as command from "@pulumi/command";
-
-const config = new pulumi.Config();
-
-//fetch credentials from proxmox-infra
-const infraRef = new pulumi.StackReference(
-  `${pulumi.getOrganization()}/proxmox-infra/dev`,
-);
-
-// Proxmox API credentials — same as proxmox-infra stack
-const pve1Endpoint = infraRef.requireOutput(
-  "pve1Endpoint",
-) as pulumi.Output<string>;
-const pve1ApiToken = infraRef.requireOutput(
-  "pve1ApiToken",
-) as pulumi.Output<string>;
-const pve2Endpoint = infraRef.requireOutput(
-  "pve2Endpoint",
-) as pulumi.Output<string>;
-const pve2ApiToken = infraRef.requireOutput(
-  "pve2ApiToken",
-) as pulumi.Output<string>;
-
-// Node IPs — static DHCP leases set in the router
-const master1Ip = config.require("master1Ip");
-const master2Ip = config.require("master2Ip");
-const master3Ip = config.require("master3Ip");
-const worker1Ip = config.require("worker1Ip");
-const worker2Ip = config.require("worker2Ip");
-
-// Pre-shared k3s cluster token
-const k3sToken = config.requireSecret("k3sToken");
-
-// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
-const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<Record<string, number>>;
-const ciRunnerPrivateKey = infraRef.requireOutput("ciRunnerPrivateKey") as pulumi.Output<string>;
-
-const master1VmId = vmIdsOutput.apply(ids => String(ids.master1));
-const master2VmId = vmIdsOutput.apply(ids => String(ids.master2));
-const master3VmId = vmIdsOutput.apply(ids => String(ids.master3));
-const worker1VmId = vmIdsOutput.apply(ids => String(ids.worker1));
-const worker2VmId = vmIdsOutput.apply(ids => String(ids.worker2));
-
-// SSH connection helper
-function conn(ip: string): command.types.input.remote.ConnectionArgs {
-    return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
-}
-
-// ---------------------------------------------------------------------------
-// Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra)
-// ---------------------------------------------------------------------------
-
-const startMaster1 = new command.local.Command("start-master-1", {
-    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`,
-    interpreter: ["/bin/bash", "-c"],
-});
-const startMaster2 = new command.local.Command("start-master-2", {
-    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`,
-    interpreter: ["/bin/bash", "-c"],
-});
-const startWorker1 = new command.local.Command("start-worker-1", {
-    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`,
-    interpreter: ["/bin/bash", "-c"],
-});
-const startMaster3 = new command.local.Command("start-master-3", {
-    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`,
-    interpreter: ["/bin/bash", "-c"],
-});
-const startWorker2 = new command.local.Command("start-worker-2", {
-    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`,
-    interpreter: ["/bin/bash", "-c"],
-});
-
-const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, startWorker2];
-
-// ---------------------------------------------------------------------------
-// Step 2: Wait for SSH on master-1, install k3s
-// ---------------------------------------------------------------------------
-
-const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", {
-    create: `for i in $(seq 1 60); do nc -z -w 5 ${master1Ip} 22 && exit 0; sleep 5; done; exit 1`,
-    interpreter: ["/bin/bash", "-c"],
-}, { dependsOn: allStarts });
-
-const installMaster1 = new command.remote.Command("install-k3s-master-1", {
-    connection: conn(master1Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip}`,
-}, { dependsOn: [waitMaster1Ssh] });
-
-const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", {
-    connection: conn(master1Ip),
-    create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`,
-}, { dependsOn: [installMaster1] });
-
-// ---------------------------------------------------------------------------
-// Step 3: Wait for SSH on remaining masters, join cluster
-// ---------------------------------------------------------------------------
-
-const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", {
-    create: `for i in $(seq 1 60); do nc -z -w 5 ${master2Ip} 22 && exit 0; sleep 5; done; exit 1`,
-    interpreter: ["/bin/bash", "-c"],
-}, { dependsOn: [waitK3sMaster1Ready] });
-
-const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", {
-    create: `for i in $(seq 1 60); do nc -z -w 5 ${master3Ip} 22 && exit 0; sleep 5; done; exit 1`,
-    interpreter: ["/bin/bash", "-c"],
-}, { dependsOn: [waitK3sMaster1Ready] });
-
-const joinMaster2 = new command.remote.Command("join-k3s-master-2", {
-    connection: conn(master2Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
-}, { dependsOn: [waitMaster2Ssh] });
-
-const joinMaster3 = new command.remote.Command("join-k3s-master-3", {
-    connection: conn(master3Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
-}, { dependsOn: [waitMaster3Ssh, joinMaster2] });
-
-// ---------------------------------------------------------------------------
-// Step 4: Join worker nodes
-// ---------------------------------------------------------------------------
-
-const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", {
-    create: `for i in $(seq 1 60); do nc -z -w 5 ${worker1Ip} 22 && exit 0; sleep 5; done; exit 1`,
-    interpreter: ["/bin/bash", "-c"],
-}, { dependsOn: [joinMaster3] });
-
-const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", {
-    create: `for i in $(seq 1 60); do nc -z -w 5 ${worker2Ip} 22 && exit 0; sleep 5; done; exit 1`,
-    interpreter: ["/bin/bash", "-c"],
-}, { dependsOn: [joinMaster3] });
-
-const joinWorker1 = new command.remote.Command("join-k3s-worker-1", {
-    connection: conn(worker1Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
-}, { dependsOn: [waitWorker1Ssh] });
-
-const joinWorker2 = new command.remote.Command("join-k3s-worker-2", {
-    connection: conn(worker2Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
-}, { dependsOn: [waitWorker2Ssh] });
-
-// ---------------------------------------------------------------------------
-// Step 5: Read kubeconfig from master-1, patch server URL for external access
-// ---------------------------------------------------------------------------
-
-const getKubeconfig = new command.remote.Command("get-kubeconfig", {
-    connection: conn(master1Ip),
-    create: `cat /etc/rancher/k3s/k3s.yaml`,
-}, { dependsOn: [joinWorker1, joinWorker2] });
-
-export const kubeconfig = pulumi.secret(
-    getKubeconfig.stdout.apply(kc => kc.replace(/127\.0\.0\.1/g, master1Ip).trim()),
-);