.gitea/workflows/deploy-k8s-bootstrap.yaml

@@ -0,0 +1,115 @@
+name: Deploy k8s Bootstrap
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'k8s-bootstrap/**'
+      - '.gitea/workflows/deploy-k8s-bootstrap.yaml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'k8s-bootstrap/**'
+      - '.gitea/workflows/deploy-k8s-bootstrap.yaml'
+
+jobs:
+  preview:
+    name: Pulumi Preview
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm install
+        working-directory: k8s-bootstrap
+
+      - name: Preview
+        uses: pulumi/actions@v5
+        with:
+          command: preview
+          stack-name: dev
+          work-dir: k8s-bootstrap
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+  deploy:
+    name: Bootstrap k3s Cluster
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+
+      - name: Restore Stack Config
+        run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
+
+      - name: Install Dependencies
+        run: npm install
+        working-directory: k8s-bootstrap
+
+      - name: Refresh State
+        uses: pulumi/actions@v5
+        with:
+          command: refresh
+          stack-name: dev
+          work-dir: k8s-bootstrap
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+      - name: Deploy
+        uses: pulumi/actions@v5
+        with:
+          command: up
+          stack-name: dev
+          work-dir: k8s-bootstrap
+          cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+      # Propagate kubeconfig to the downstream stacks so their next deploy picks it up
+      - name: Propagate kubeconfig to k8s-infra
+        run: |
+          echo "${{ secrets.K8S_INFRA_DEV_YAML }}" | base64 -d > k8s-infra/Pulumi.dev.yaml
+          cd k8s-infra && npm install
+          KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \
+            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev)
+          pulumi config set --secret kubeconfig "$KUBECONFIG" \
+            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev
+          # Re-encode updated config for the secret (update manually in Gitea after first run)
+          base64 -w 0 Pulumi.dev.yaml
+        working-directory: .
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
+
+      - name: Propagate kubeconfig to k8s-apps
+        run: |
+          echo "${{ secrets.K8S_APPS_DEV_YAML }}" | base64 -d > k8s-apps/Pulumi.dev.yaml
+          cd k8s-apps && npm install
+          KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \
+            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev)
+          pulumi config set --secret kubeconfig "$KUBECONFIG" \
+            --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev
+          base64 -w 0 Pulumi.dev.yaml
+        working-directory: .
+        env:
+          PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}

k8s-bootstrap/CLAUDE.md

@@ -0,0 +1,85 @@
+# k8s-bootstrap
+
+Bootstraps a k3s cluster on the 5 Proxmox VMs created by `proxmox-infra`. Starts VMs via the Proxmox REST API, then provisions k3s over SSH using `@pulumi/command` remote.Command.
+
+## How it works
+
+1. Starts all 5 VMs via `POST /api2/json/nodes/{node}/qemu/{vmid}/status/start`
+2. Waits for port 22 to open on each VM (`nc -z`)
+3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
+4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
+5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes
+6. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches server URL, exports as secret stack output
+
+VM IDs and the CI runner SSH private key are read automatically from the `proxmox-infra` stack output via `StackReference` — no manual setup needed for those.
+
+## Required Pulumi config
+
+Run in this directory after `pulumi stack init dev`:
+
+```bash
+# Same as proxmox-infra
+pulumi config set --secret pve1Endpoint "https://192.168.1.x:8006"
+pulumi config set --secret pve1ApiToken "root@pam!pulumi=<token>"
+pulumi config set --secret pve2Endpoint "https://192.168.1.y:8006"
+pulumi config set --secret pve2ApiToken "root@pam!pulumi=<token>"
+
+# Pre-shared k3s token — any strong random string
+pulumi config set --secret k3sToken "$(openssl rand -hex 32)"
+
+# Node IPs — static DHCP leases from the router (not secrets)
+pulumi config set master1Ip "192.168.1.x"
+pulumi config set master2Ip "192.168.1.x"
+pulumi config set master3Ip "192.168.1.x"
+pulumi config set worker1Ip "192.168.1.x"
+pulumi config set worker2Ip "192.168.1.x"
+```
+
+After setting config, re-encode `Pulumi.dev.yaml` and update the Gitea secret `K8S_BOOTSTRAP_DEV_YAML`:
+```bash
+base64 -w 0 Pulumi.dev.yaml
+# Copy output → Gitea → Settings → Actions → Secrets → K8S_BOOTSTRAP_DEV_YAML
+```
+
+## Prerequisites
+
+`proxmox-infra` must be deployed first. On first `pulumi up`, it generates an ed25519 SSH keypair via `@pulumi/tls`, writes the public key into every VM's cloud-init, and exports the private key as a secret stack output. VMs must be re-provisioned (or the public key manually added to `~/.ssh/authorized_keys`) before k8s-bootstrap can SSH in.
+
+## After first run
+
+The `kubeconfig` stack output must be propagated to `k8s-infra` and `k8s-apps`:
+
+```bash
+KUBECONFIG=$(pulumi stack output kubeconfig --show-secrets)
+cd ../k8s-infra && pulumi config set --secret kubeconfig "$KUBECONFIG"
+cd ../k8s-apps  && pulumi config set --secret kubeconfig "$KUBECONFIG"
+```
+
+Then re-encode each `Pulumi.dev.yaml` and update the corresponding Gitea secrets
+(`K8S_INFRA_DEV_YAML`, `K8S_APPS_DEV_YAML`):
+
+```bash
+base64 -w 0 k8s-infra/Pulumi.dev.yaml
+base64 -w 0 k8s-apps/Pulumi.dev.yaml
+```
+
+## k8s-infra config changes (democratic-csi → NFS CSI)
+
+The `k8s-infra` stack no longer needs `truenasApiKey` or `truenasDataset`.
+Replace them with:
+
+```bash
+cd ../k8s-infra
+pulumi config set truenasHost "192.168.1.x"
+pulumi config set truenasNfsPath "/mnt/tank/k8s"
+# Remove old keys if present:
+pulumi config rm truenasApiKey
+pulumi config rm truenasDataset
+```
+
+## TrueNAS one-time setup (before deploying k8s-infra)
+
+1. Create a dataset: `tank/k8s`
+2. Add an NFS share for that dataset
+3. In Network → Allowed Networks, permit `192.168.1.0/24`
+4. No API key required — the NFS CSI driver connects directly via NFS protocol

k8s-bootstrap/Pulumi.yaml

@@ -0,0 +1,10 @@
+name: k8s-bootstrap
+description: Bootstrap k3s cluster on Proxmox VMs via QEMU guest exec
+runtime:
+  name: nodejs
+  options:
+    packagemanager: npm
+config:
+  pulumi:tags:
+    value:
+      pulumi:template: typescript

k8s-bootstrap/index.ts

@@ -0,0 +1,155 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as command from "@pulumi/command";
+
+const config = new pulumi.Config();
+
+//fetch credentials from proxmox-infra
+const infraRef = new pulumi.StackReference(
+  `${pulumi.getOrganization()}/proxmox-infra/dev`,
+);
+
+// Proxmox API credentials — same as proxmox-infra stack
+const pve1Endpoint = infraRef.requireOutput(
+  "pve1Endpoint",
+) as pulumi.Output<string>;
+const pve1ApiToken = infraRef.requireOutput(
+  "pve1ApiToken",
+) as pulumi.Output<string>;
+const pve2Endpoint = infraRef.requireOutput(
+  "pve2Endpoint",
+) as pulumi.Output<string>;
+const pve2ApiToken = infraRef.requireOutput(
+  "pve2ApiToken",
+) as pulumi.Output<string>;
+
+// Node IPs — static DHCP leases set in the router
+const master1Ip = config.require("master1Ip");
+const master2Ip = config.require("master2Ip");
+const master3Ip = config.require("master3Ip");
+const worker1Ip = config.require("worker1Ip");
+const worker2Ip = config.require("worker2Ip");
+
+// Pre-shared k3s cluster token
+const k3sToken = config.requireSecret("k3sToken");
+
+// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
+const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<Record<string, number>>;
+const ciRunnerPrivateKey = infraRef.requireOutput("ciRunnerPrivateKey") as pulumi.Output<string>;
+
+const master1VmId = vmIdsOutput.apply(ids => String(ids.master1));
+const master2VmId = vmIdsOutput.apply(ids => String(ids.master2));
+const master3VmId = vmIdsOutput.apply(ids => String(ids.master3));
+const worker1VmId = vmIdsOutput.apply(ids => String(ids.worker1));
+const worker2VmId = vmIdsOutput.apply(ids => String(ids.worker2));
+
+// SSH connection helper
+function conn(ip: string): command.types.input.remote.ConnectionArgs {
+    return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
+}
+
+// ---------------------------------------------------------------------------
+// Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra)
+// ---------------------------------------------------------------------------
+
+const startMaster1 = new command.local.Command("start-master-1", {
+    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`,
+    interpreter: ["/bin/bash", "-c"],
+});
+const startMaster2 = new command.local.Command("start-master-2", {
+    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`,
+    interpreter: ["/bin/bash", "-c"],
+});
+const startWorker1 = new command.local.Command("start-worker-1", {
+    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`,
+    interpreter: ["/bin/bash", "-c"],
+});
+const startMaster3 = new command.local.Command("start-master-3", {
+    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`,
+    interpreter: ["/bin/bash", "-c"],
+});
+const startWorker2 = new command.local.Command("start-worker-2", {
+    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`,
+    interpreter: ["/bin/bash", "-c"],
+});
+
+const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, startWorker2];
+
+// ---------------------------------------------------------------------------
+// Step 2: Wait for SSH on master-1, install k3s
+// ---------------------------------------------------------------------------
+
+const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", {
+    create: `for i in $(seq 1 60); do nc -z -w 5 ${master1Ip} 22 && exit 0; sleep 5; done; exit 1`,
+    interpreter: ["/bin/bash", "-c"],
+}, { dependsOn: allStarts });
+
+const installMaster1 = new command.remote.Command("install-k3s-master-1", {
+    connection: conn(master1Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip}`,
+}, { dependsOn: [waitMaster1Ssh] });
+
+const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", {
+    connection: conn(master1Ip),
+    create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`,
+}, { dependsOn: [installMaster1] });
+
+// ---------------------------------------------------------------------------
+// Step 3: Wait for SSH on remaining masters, join cluster
+// ---------------------------------------------------------------------------
+
+const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", {
+    create: `for i in $(seq 1 60); do nc -z -w 5 ${master2Ip} 22 && exit 0; sleep 5; done; exit 1`,
+    interpreter: ["/bin/bash", "-c"],
+}, { dependsOn: [waitK3sMaster1Ready] });
+
+const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", {
+    create: `for i in $(seq 1 60); do nc -z -w 5 ${master3Ip} 22 && exit 0; sleep 5; done; exit 1`,
+    interpreter: ["/bin/bash", "-c"],
+}, { dependsOn: [waitK3sMaster1Ready] });
+
+const joinMaster2 = new command.remote.Command("join-k3s-master-2", {
+    connection: conn(master2Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
+}, { dependsOn: [waitMaster2Ssh] });
+
+const joinMaster3 = new command.remote.Command("join-k3s-master-3", {
+    connection: conn(master3Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
+}, { dependsOn: [waitMaster3Ssh, joinMaster2] });
+
+// ---------------------------------------------------------------------------
+// Step 4: Join worker nodes
+// ---------------------------------------------------------------------------
+
+const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", {
+    create: `for i in $(seq 1 60); do nc -z -w 5 ${worker1Ip} 22 && exit 0; sleep 5; done; exit 1`,
+    interpreter: ["/bin/bash", "-c"],
+}, { dependsOn: [joinMaster3] });
+
+const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", {
+    create: `for i in $(seq 1 60); do nc -z -w 5 ${worker2Ip} 22 && exit 0; sleep 5; done; exit 1`,
+    interpreter: ["/bin/bash", "-c"],
+}, { dependsOn: [joinMaster3] });
+
+const joinWorker1 = new command.remote.Command("join-k3s-worker-1", {
+    connection: conn(worker1Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
+}, { dependsOn: [waitWorker1Ssh] });
+
+const joinWorker2 = new command.remote.Command("join-k3s-worker-2", {
+    connection: conn(worker2Ip),
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
+}, { dependsOn: [waitWorker2Ssh] });
+
+// ---------------------------------------------------------------------------
+// Step 5: Read kubeconfig from master-1, patch server URL for external access
+// ---------------------------------------------------------------------------
+
+const getKubeconfig = new command.remote.Command("get-kubeconfig", {
+    connection: conn(master1Ip),
+    create: `cat /etc/rancher/k3s/k3s.yaml`,
+}, { dependsOn: [joinWorker1, joinWorker2] });
+
+export const kubeconfig = pulumi.secret(
+    getKubeconfig.stdout.apply(kc => kc.replace(/127\.0\.0\.1/g, master1Ip).trim()),
+);

k8s-bootstrap/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "k8s-bootstrap",
+    "main": "index.ts",
+    "devDependencies": {
+        "@types/node": "^18",
+        "typescript": "^5.0.0"
+    },
+    "dependencies": {
+        "@pulumi/command": "^0.11.0",
+        "@pulumi/pulumi": "^3.113.0"
+    }
+}

k8s-bootstrap/tsconfig.json

@@ -0,0 +1,18 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "outDir": "bin",
+        "target": "es2020",
+        "module": "nodenext",
+        "moduleResolution": "nodenext",
+        "sourceMap": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}

proxmox-infra/index.ts

@@ -1,5 +1,6 @@
 import * as pulumi from "@pulumi/pulumi";
 import * as proxmox from "@muhlba91/pulumi-proxmoxve";
+import * as tls from "@pulumi/tls";
 
 const config = new pulumi.Config();
 
@@ -7,18 +8,32 @@ const config = new pulumi.Config();
 // Providers — one per standalone Proxmox machine
 // ---------------------------------------------------------------------------
 
+const pve1Endpoint = config.requireSecret("pve1Endpoint");
+const pve1ApiToken = config.requireSecret("pve1ApiToken");
+const pve2Endpoint = config.requireSecret("pve2Endpoint");
+const pve2ApiToken = config.requireSecret("pve2ApiToken");
+
 const pveProvider = new proxmox.Provider("pve", {
-  endpoint: config.requireSecret("pve1Endpoint"),
-  apiToken: config.requireSecret("pve1ApiToken"),
+  endpoint: pve1Endpoint,
+  apiToken: pve1ApiToken,
   insecure: true,
 });
 
 const pveBckpProvider = new proxmox.Provider("pve-bckp", {
-  endpoint: config.requireSecret("pve2Endpoint"),
-  apiToken: config.requireSecret("pve2ApiToken"),
+  endpoint: pve2Endpoint,
+  apiToken: pve2ApiToken,
   insecure: true,
 });
 
+// ---------------------------------------------------------------------------
+// CI runner SSH keypair — generated once, stored in Pulumi state backend.
+// Public key goes into every VM; private key is exported for k8s-bootstrap.
+// ---------------------------------------------------------------------------
+
+const ciRunnerKey = new tls.PrivateKey("ci-runner-key", {
+  algorithm: "ED25519",
+});
+
 // ---------------------------------------------------------------------------
 // Download Ubuntu Noble cloud image to each node's ISO storage
 // ---------------------------------------------------------------------------
@@ -219,7 +234,10 @@ const k3sVms = nodeConfigs.map(
           userAccount: {
             username: "ubuntu",
             password: k3sVmPassword,
-            keys: [sshPvePublicKey.apply((k) => k.trim())],
+            keys: [
+                  sshPvePublicKey.apply((k) => k.trim()),
+                  ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
+                ],
           },
         },
         networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
@@ -245,7 +263,7 @@ export const clusterInfo = k3sVms.map((vm, index) => ({
   role: nodeConfigs[index].role,
 }));
 
-// Individual vmId exports — used by k8s-bootstrap to start VMs and run guest exec.
+// Individual vmId exports — used by k8s-bootstrap to start VMs.
 // Order matches nodeConfigs: master-1, master-2, worker-1, master-3, worker-2.
 export const vmIds = {
   master1: k3sVms[0].vmId,
@@ -254,3 +272,9 @@ export const vmIds = {
   master3: k3sVms[3].vmId,
   worker2: k3sVms[4].vmId,
 };
+
+// CI runner SSH private key — consumed by k8s-bootstrap via StackReference.
+export const ciRunnerPrivateKey = pulumi.secret(ciRunnerKey.privateKeyOpenssh);
+
+// Proxmox API credentials — consumed by k8s-bootstrap via StackReference.
+export { pve1Endpoint, pve1ApiToken, pve2Endpoint, pve2ApiToken };

proxmox-infra/package-lock.json

@@ -7,7 +7,8 @@
             "name": "proxmox-infra",
             "dependencies": {
                 "@muhlba91/pulumi-proxmoxve": "^8.2.1",
-                "@pulumi/pulumi": "^3.113.0"
+                "@pulumi/pulumi": "^3.113.0",
+                "@pulumi/tls": "^5.5.0"
             },
             "devDependencies": {
                 "@types/node": "^18",
@@ -752,6 +753,15 @@
                 }
             }
         },
+        "node_modules/@pulumi/tls": {
+            "version": "5.5.0",
+            "resolved": "https://registry.npmjs.org/@pulumi/tls/-/tls-5.5.0.tgz",
+            "integrity": "sha512-OTGxp4sgDEuXlXrd7NtxrhhciPgcn9rqDIZlGOTmTaeGo+tlVMwv73FFqiVrzAmBiILdU8tLXHfxhjc+bTLdrQ==",
+            "license": "Apache-2.0",
+            "dependencies": {
+                "@pulumi/pulumi": "^3.142.0"
+            }
+        },
         "node_modules/@sigstore/bundle": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/@sigstore/bundle/-/bundle-4.0.0.tgz",

proxmox-infra/package.json

@@ -7,6 +7,7 @@
     },
     "dependencies": {
         "@muhlba91/pulumi-proxmoxve": "^8.2.1",
-        "@pulumi/pulumi": "^3.113.0"
+        "@pulumi/pulumi": "^3.113.0",
+        "@pulumi/tls": "^5.5.0"
     }
 }