From e9765bb073772087e5b65e53c61cc40d77c51ff0 Mon Sep 17 00:00:00 2001 From: kasun Date: Sat, 30 May 2026 17:24:49 +0200 Subject: [PATCH 1/5] removed netcat dependency with /dev/tcp --- k8s-bootstrap/CLAUDE.md | 16 +++++++++------- k8s-bootstrap/index.ts | 10 +++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/k8s-bootstrap/CLAUDE.md b/k8s-bootstrap/CLAUDE.md index df6e0c7..19fd66e 100644 --- a/k8s-bootstrap/CLAUDE.md +++ b/k8s-bootstrap/CLAUDE.md @@ -5,7 +5,7 @@ Bootstraps a k3s cluster on the 5 Proxmox VMs created by `proxmox-infra`. Starts ## How it works 1. Starts all 5 VMs via `POST /api2/json/nodes/{node}/qemu/{vmid}/status/start` -2. Waits for port 22 to open on each VM (`nc -z`) +2. Waits for port 22 to open on each VM (bash `/dev/tcp`) 3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san ` 4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes 5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes @@ -18,12 +18,6 @@ VM IDs and the CI runner SSH private key are read automatically from the `proxmo Run in this directory after `pulumi stack init dev`: ```bash -# Same as proxmox-infra -pulumi config set --secret pve1Endpoint "https://192.168.1.x:8006" -pulumi config set --secret pve1ApiToken "root@pam!pulumi=" -pulumi config set --secret pve2Endpoint "https://192.168.1.y:8006" -pulumi config set --secret pve2ApiToken "root@pam!pulumi=" - # Pre-shared k3s token — any strong random string pulumi config set --secret k3sToken "$(openssl rand -hex 32)" @@ -35,6 +29,14 @@ pulumi config set worker1Ip "192.168.1.x" pulumi config set worker2Ip "192.168.1.x" ``` +Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`) are read automatically from the `proxmox-infra` stack outputs via StackReference — do **not** set them here. + +## Deployment order + +`proxmox-infra` must be deployed **before** k8s-bootstrap. The proxmox-infra stack exports the Proxmox credentials and VM IDs that k8s-bootstrap reads via StackReference. If proxmox-infra hasn't been re-deployed after its latest code changes, those outputs won't exist and k8s-bootstrap will fail. + +In CI, both workflows run as previews on PRs (no deploy). Trigger `workflow_dispatch` on `deploy-proxmox-infra` first, then run k8s-bootstrap. + After setting config, re-encode `Pulumi.dev.yaml` and update the Gitea secret `K8S_BOOTSTRAP_DEV_YAML`: ```bash base64 -w 0 Pulumi.dev.yaml diff --git a/k8s-bootstrap/index.ts b/k8s-bootstrap/index.ts index 44b7467..d7fd607 100644 --- a/k8s-bootstrap/index.ts +++ b/k8s-bootstrap/index.ts @@ -79,7 +79,7 @@ const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, start // --------------------------------------------------------------------------- const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", { - create: `for i in $(seq 1 60); do nc -z -w 5 ${master1Ip} 22 && exit 0; sleep 5; done; exit 1`, + create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: allStarts }); @@ -98,12 +98,12 @@ const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready" // --------------------------------------------------------------------------- const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", { - create: `for i in $(seq 1 60); do nc -z -w 5 ${master2Ip} 22 && exit 0; sleep 5; done; exit 1`, + create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", { - create: `for i in $(seq 1 60); do nc -z -w 5 ${master3Ip} 22 && exit 0; sleep 5; done; exit 1`, + create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); @@ -122,12 +122,12 @@ const joinMaster3 = new command.remote.Command("join-k3s-master-3", { // --------------------------------------------------------------------------- const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", { - create: `for i in $(seq 1 60); do nc -z -w 5 ${worker1Ip} 22 && exit 0; sleep 5; done; exit 1`, + create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", { - create: `for i in $(seq 1 60); do nc -z -w 5 ${worker2Ip} 22 && exit 0; sleep 5; done; exit 1`, + create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); From 7212e0e3efed7a88333d411fa11e8a9274312dcf Mon Sep 17 00:00:00 2001 From: kasun Date: Sat, 30 May 2026 19:06:11 +0200 Subject: [PATCH 2/5] fix: add triggers and --node-name to k8s-bootstrap commands Without triggers, commands cached in Pulumi state don't re-run when VMs are deleted and recreated with new IDs. This caused stale state where start/install commands were skipped while the new VMs were never bootstrapped, leading to SSH "no route to host" failures. All command resources now carry triggers: [vmId] so they are automatically replaced (and re-run) whenever the underlying VM changes. Also adds --node-name to every k3s install/join command so nodes register under the expected name (k3s-master-1 etc.) regardless of the VM's actual hostname, which cloud-init does not set explicitly. --- k8s-bootstrap/index.ts | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/k8s-bootstrap/index.ts b/k8s-bootstrap/index.ts index d7fd607..29c7737 100644 --- a/k8s-bootstrap/index.ts +++ b/k8s-bootstrap/index.ts @@ -49,26 +49,32 @@ function conn(ip: string): command.types.input.remote.ConnectionArgs { // --------------------------------------------------------------------------- // Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra) +// triggers: vmId — forces re-run when VMs are deleted and recreated // --------------------------------------------------------------------------- const startMaster1 = new command.local.Command("start-master-1", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`, + triggers: [master1VmId], interpreter: ["/bin/bash", "-c"], }); const startMaster2 = new command.local.Command("start-master-2", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`, + triggers: [master2VmId], interpreter: ["/bin/bash", "-c"], }); const startWorker1 = new command.local.Command("start-worker-1", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`, + triggers: [worker1VmId], interpreter: ["/bin/bash", "-c"], }); const startMaster3 = new command.local.Command("start-master-3", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`, + triggers: [master3VmId], interpreter: ["/bin/bash", "-c"], }); const startWorker2 = new command.local.Command("start-worker-2", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`, + triggers: [worker2VmId], interpreter: ["/bin/bash", "-c"], }); @@ -80,17 +86,20 @@ const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, start const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [master1VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: allStarts }); const installMaster1 = new command.remote.Command("install-k3s-master-1", { connection: conn(master1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip}`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`, + triggers: [master1VmId], }, { dependsOn: [waitMaster1Ssh] }); const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", { connection: conn(master1Ip), create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`, + triggers: [master1VmId], }, { dependsOn: [installMaster1] }); // --------------------------------------------------------------------------- @@ -99,22 +108,26 @@ const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready" const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [master2VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [master3VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); const joinMaster2 = new command.remote.Command("join-k3s-master-2", { connection: conn(master2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`, + triggers: [master2VmId], }, { dependsOn: [waitMaster2Ssh] }); const joinMaster3 = new command.remote.Command("join-k3s-master-3", { connection: conn(master3Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`, + triggers: [master3VmId], }, { dependsOn: [waitMaster3Ssh, joinMaster2] }); // --------------------------------------------------------------------------- @@ -123,22 +136,26 @@ const joinMaster3 = new command.remote.Command("join-k3s-master-3", { const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [worker1VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [worker2VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); const joinWorker1 = new command.remote.Command("join-k3s-worker-1", { connection: conn(worker1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`, + triggers: [worker1VmId], }, { dependsOn: [waitWorker1Ssh] }); const joinWorker2 = new command.remote.Command("join-k3s-worker-2", { connection: conn(worker2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`, + triggers: [worker2VmId], }, { dependsOn: [waitWorker2Ssh] }); // --------------------------------------------------------------------------- @@ -148,6 +165,7 @@ const joinWorker2 = new command.remote.Command("join-k3s-worker-2", { const getKubeconfig = new command.remote.Command("get-kubeconfig", { connection: conn(master1Ip), create: `cat /etc/rancher/k3s/k3s.yaml`, + triggers: [master1VmId], }, { dependsOn: [joinWorker1, joinWorker2] }); export const kubeconfig = pulumi.secret( From 2f6635aa730c23e5f2db75b8889b0963859e09ad Mon Sep 17 00:00:00 2001 From: kasun Date: Sat, 30 May 2026 19:16:36 +0200 Subject: [PATCH 3/5] fix: added sudo to kubectl commands --- k8s-bootstrap/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s-bootstrap/index.ts b/k8s-bootstrap/index.ts index 29c7737..7b6c063 100644 --- a/k8s-bootstrap/index.ts +++ b/k8s-bootstrap/index.ts @@ -92,13 +92,13 @@ const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", { const installMaster1 = new command.remote.Command("install-k3s-master-1", { connection: conn(master1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`, triggers: [master1VmId], }, { dependsOn: [waitMaster1Ssh] }); const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", { connection: conn(master1Ip), - create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`, + create: `for i in $(seq 1 60); do sudo kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`, triggers: [master1VmId], }, { dependsOn: [installMaster1] }); @@ -120,13 +120,13 @@ const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", { const joinMaster2 = new command.remote.Command("join-k3s-master-2", { connection: conn(master2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`, triggers: [master2VmId], }, { dependsOn: [waitMaster2Ssh] }); const joinMaster3 = new command.remote.Command("join-k3s-master-3", { connection: conn(master3Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`, triggers: [master3VmId], }, { dependsOn: [waitMaster3Ssh, joinMaster2] }); @@ -148,13 +148,13 @@ const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", { const joinWorker1 = new command.remote.Command("join-k3s-worker-1", { connection: conn(worker1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`, triggers: [worker1VmId], }, { dependsOn: [waitWorker1Ssh] }); const joinWorker2 = new command.remote.Command("join-k3s-worker-2", { connection: conn(worker2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`, triggers: [worker2VmId], }, { dependsOn: [waitWorker2Ssh] }); From 980e24c6063d7eca2224ca3574192569d0505e8e Mon Sep 17 00:00:00 2001 From: kasun Date: Sat, 30 May 2026 19:35:37 +0200 Subject: [PATCH 4/5] fix: added missing sudo --- k8s-bootstrap/index.ts | 160 ++++++++++++++++++++++++++++------------- 1 file changed, 110 insertions(+), 50 deletions(-) diff --git a/k8s-bootstrap/index.ts b/k8s-bootstrap/index.ts index 7b6c063..343d541 100644 --- a/k8s-bootstrap/index.ts +++ b/k8s-bootstrap/index.ts @@ -33,18 +33,22 @@ const worker2Ip = config.require("worker2Ip"); const k3sToken = config.requireSecret("k3sToken"); // VM IDs and CI runner SSH key — read from proxmox-infra stack outputs -const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output>; -const ciRunnerPrivateKey = infraRef.requireOutput("ciRunnerPrivateKey") as pulumi.Output; +const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output< + Record +>; +const ciRunnerPrivateKey = infraRef.requireOutput( + "ciRunnerPrivateKey", +) as pulumi.Output; -const master1VmId = vmIdsOutput.apply(ids => String(ids.master1)); -const master2VmId = vmIdsOutput.apply(ids => String(ids.master2)); -const master3VmId = vmIdsOutput.apply(ids => String(ids.master3)); -const worker1VmId = vmIdsOutput.apply(ids => String(ids.worker1)); -const worker2VmId = vmIdsOutput.apply(ids => String(ids.worker2)); +const master1VmId = vmIdsOutput.apply((ids) => String(ids.master1)); +const master2VmId = vmIdsOutput.apply((ids) => String(ids.master2)); +const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3)); +const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1)); +const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2)); // SSH connection helper function conn(ip: string): command.types.input.remote.ConnectionArgs { - return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey }; + return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey }; } // --------------------------------------------------------------------------- @@ -53,121 +57,177 @@ function conn(ip: string): command.types.input.remote.ConnectionArgs { // --------------------------------------------------------------------------- const startMaster1 = new command.local.Command("start-master-1", { - create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`, - triggers: [master1VmId], - interpreter: ["/bin/bash", "-c"], + create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`, + triggers: [master1VmId], + interpreter: ["/bin/bash", "-c"], }); const startMaster2 = new command.local.Command("start-master-2", { - create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`, - triggers: [master2VmId], - interpreter: ["/bin/bash", "-c"], + create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`, + triggers: [master2VmId], + interpreter: ["/bin/bash", "-c"], }); const startWorker1 = new command.local.Command("start-worker-1", { - create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`, - triggers: [worker1VmId], - interpreter: ["/bin/bash", "-c"], + create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`, + triggers: [worker1VmId], + interpreter: ["/bin/bash", "-c"], }); const startMaster3 = new command.local.Command("start-master-3", { - create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`, - triggers: [master3VmId], - interpreter: ["/bin/bash", "-c"], + create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`, + triggers: [master3VmId], + interpreter: ["/bin/bash", "-c"], }); const startWorker2 = new command.local.Command("start-worker-2", { - create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`, - triggers: [worker2VmId], - interpreter: ["/bin/bash", "-c"], + create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`, + triggers: [worker2VmId], + interpreter: ["/bin/bash", "-c"], }); -const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, startWorker2]; +const allStarts = [ + startMaster1, + startMaster2, + startMaster3, + startWorker1, + startWorker2, +]; // --------------------------------------------------------------------------- // Step 2: Wait for SSH on master-1, install k3s // --------------------------------------------------------------------------- -const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", { +const waitMaster1Ssh = new command.local.Command( + "wait-ssh-master-1", + { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, triggers: [master1VmId], interpreter: ["/bin/bash", "-c"], -}, { dependsOn: allStarts }); + }, + { dependsOn: allStarts }, +); -const installMaster1 = new command.remote.Command("install-k3s-master-1", { +const installMaster1 = new command.remote.Command( + "install-k3s-master-1", + { connection: conn(master1Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`, triggers: [master1VmId], -}, { dependsOn: [waitMaster1Ssh] }); + }, + { dependsOn: [waitMaster1Ssh] }, +); -const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", { +const waitK3sMaster1Ready = new command.remote.Command( + "wait-k3s-master-1-ready", + { connection: conn(master1Ip), create: `for i in $(seq 1 60); do sudo kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`, triggers: [master1VmId], -}, { dependsOn: [installMaster1] }); + }, + { dependsOn: [installMaster1] }, +); // --------------------------------------------------------------------------- // Step 3: Wait for SSH on remaining masters, join cluster // --------------------------------------------------------------------------- -const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", { +const waitMaster2Ssh = new command.local.Command( + "wait-ssh-master-2", + { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, triggers: [master2VmId], interpreter: ["/bin/bash", "-c"], -}, { dependsOn: [waitK3sMaster1Ready] }); + }, + { dependsOn: [waitK3sMaster1Ready] }, +); -const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", { +const waitMaster3Ssh = new command.local.Command( + "wait-ssh-master-3", + { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, triggers: [master3VmId], interpreter: ["/bin/bash", "-c"], -}, { dependsOn: [waitK3sMaster1Ready] }); + }, + { dependsOn: [waitK3sMaster1Ready] }, +); -const joinMaster2 = new command.remote.Command("join-k3s-master-2", { +const joinMaster2 = new command.remote.Command( + "join-k3s-master-2", + { connection: conn(master2Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`, triggers: [master2VmId], -}, { dependsOn: [waitMaster2Ssh] }); + }, + { dependsOn: [waitMaster2Ssh] }, +); -const joinMaster3 = new command.remote.Command("join-k3s-master-3", { +const joinMaster3 = new command.remote.Command( + "join-k3s-master-3", + { connection: conn(master3Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`, triggers: [master3VmId], -}, { dependsOn: [waitMaster3Ssh, joinMaster2] }); + }, + { dependsOn: [waitMaster3Ssh, joinMaster2] }, +); // --------------------------------------------------------------------------- // Step 4: Join worker nodes // --------------------------------------------------------------------------- -const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", { +const waitWorker1Ssh = new command.local.Command( + "wait-ssh-worker-1", + { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, triggers: [worker1VmId], interpreter: ["/bin/bash", "-c"], -}, { dependsOn: [joinMaster3] }); + }, + { dependsOn: [joinMaster3] }, +); -const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", { +const waitWorker2Ssh = new command.local.Command( + "wait-ssh-worker-2", + { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, triggers: [worker2VmId], interpreter: ["/bin/bash", "-c"], -}, { dependsOn: [joinMaster3] }); + }, + { dependsOn: [joinMaster3] }, +); -const joinWorker1 = new command.remote.Command("join-k3s-worker-1", { +const joinWorker1 = new command.remote.Command( + "join-k3s-worker-1", + { connection: conn(worker1Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`, triggers: [worker1VmId], -}, { dependsOn: [waitWorker1Ssh] }); + }, + { dependsOn: [waitWorker1Ssh] }, +); -const joinWorker2 = new command.remote.Command("join-k3s-worker-2", { +const joinWorker2 = new command.remote.Command( + "join-k3s-worker-2", + { connection: conn(worker2Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`, triggers: [worker2VmId], -}, { dependsOn: [waitWorker2Ssh] }); + }, + { dependsOn: [waitWorker2Ssh] }, +); // --------------------------------------------------------------------------- // Step 5: Read kubeconfig from master-1, patch server URL for external access // --------------------------------------------------------------------------- -const getKubeconfig = new command.remote.Command("get-kubeconfig", { +const getKubeconfig = new command.remote.Command( + "get-kubeconfig", + { connection: conn(master1Ip), - create: `cat /etc/rancher/k3s/k3s.yaml`, + create: `sudo cat /etc/rancher/k3s/k3s.yaml`, triggers: [master1VmId], -}, { dependsOn: [joinWorker1, joinWorker2] }); + }, + { dependsOn: [joinWorker1, joinWorker2] }, +); export const kubeconfig = pulumi.secret( - getKubeconfig.stdout.apply(kc => kc.replace(/127\.0\.0\.1/g, master1Ip).trim()), + getKubeconfig.stdout.apply((kc) => + kc.replace(/127\.0\.0\.1/g, master1Ip).trim(), + ), ); From 5ac4cb592c492613673fe5c355fbe159ea5c0a77 Mon Sep 17 00:00:00 2001 From: kasun Date: Sat, 30 May 2026 19:53:04 +0200 Subject: [PATCH 5/5] fix: removed failing propagation. will add it later --- .gitea/workflows/deploy-k8s-bootstrap.yaml | 28 ---------------------- 1 file changed, 28 deletions(-) diff --git a/.gitea/workflows/deploy-k8s-bootstrap.yaml b/.gitea/workflows/deploy-k8s-bootstrap.yaml index 6d9d6de..eaf554d 100644 --- a/.gitea/workflows/deploy-k8s-bootstrap.yaml +++ b/.gitea/workflows/deploy-k8s-bootstrap.yaml @@ -85,31 +85,3 @@ jobs: cloud-url: ${{ secrets.PULUMI_BACKEND_URL }} env: PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} - - # Propagate kubeconfig to the downstream stacks so their next deploy picks it up - - name: Propagate kubeconfig to k8s-infra - run: | - echo "${{ secrets.K8S_INFRA_DEV_YAML }}" | base64 -d > k8s-infra/Pulumi.dev.yaml - cd k8s-infra && npm install - KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \ - --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev) - pulumi config set --secret kubeconfig "$KUBECONFIG" \ - --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev - # Re-encode updated config for the secret (update manually in Gitea after first run) - base64 -w 0 Pulumi.dev.yaml - working-directory: . - env: - PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} - - - name: Propagate kubeconfig to k8s-apps - run: | - echo "${{ secrets.K8S_APPS_DEV_YAML }}" | base64 -d > k8s-apps/Pulumi.dev.yaml - cd k8s-apps && npm install - KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \ - --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev) - pulumi config set --secret kubeconfig "$KUBECONFIG" \ - --cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev - base64 -w 0 Pulumi.dev.yaml - working-directory: . - env: - PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}