From 7212e0e3efed7a88333d411fa11e8a9274312dcf Mon Sep 17 00:00:00 2001 From: kasun Date: Sat, 30 May 2026 19:06:11 +0200 Subject: [PATCH] fix: add triggers and --node-name to k8s-bootstrap commands Without triggers, commands cached in Pulumi state don't re-run when VMs are deleted and recreated with new IDs. This caused stale state where start/install commands were skipped while the new VMs were never bootstrapped, leading to SSH "no route to host" failures. All command resources now carry triggers: [vmId] so they are automatically replaced (and re-run) whenever the underlying VM changes. Also adds --node-name to every k3s install/join command so nodes register under the expected name (k3s-master-1 etc.) regardless of the VM's actual hostname, which cloud-init does not set explicitly. --- k8s-bootstrap/index.ts | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/k8s-bootstrap/index.ts b/k8s-bootstrap/index.ts index d7fd607..29c7737 100644 --- a/k8s-bootstrap/index.ts +++ b/k8s-bootstrap/index.ts @@ -49,26 +49,32 @@ function conn(ip: string): command.types.input.remote.ConnectionArgs { // --------------------------------------------------------------------------- // Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra) +// triggers: vmId — forces re-run when VMs are deleted and recreated // --------------------------------------------------------------------------- const startMaster1 = new command.local.Command("start-master-1", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`, + triggers: [master1VmId], interpreter: ["/bin/bash", "-c"], }); const startMaster2 = new command.local.Command("start-master-2", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`, + triggers: [master2VmId], interpreter: ["/bin/bash", "-c"], }); const startWorker1 = new command.local.Command("start-worker-1", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`, + triggers: [worker1VmId], interpreter: ["/bin/bash", "-c"], }); const startMaster3 = new command.local.Command("start-master-3", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`, + triggers: [master3VmId], interpreter: ["/bin/bash", "-c"], }); const startWorker2 = new command.local.Command("start-worker-2", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`, + triggers: [worker2VmId], interpreter: ["/bin/bash", "-c"], }); @@ -80,17 +86,20 @@ const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, start const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [master1VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: allStarts }); const installMaster1 = new command.remote.Command("install-k3s-master-1", { connection: conn(master1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip}`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`, + triggers: [master1VmId], }, { dependsOn: [waitMaster1Ssh] }); const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", { connection: conn(master1Ip), create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`, + triggers: [master1VmId], }, { dependsOn: [installMaster1] }); // --------------------------------------------------------------------------- @@ -99,22 +108,26 @@ const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready" const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [master2VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [master3VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); const joinMaster2 = new command.remote.Command("join-k3s-master-2", { connection: conn(master2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`, + triggers: [master2VmId], }, { dependsOn: [waitMaster2Ssh] }); const joinMaster3 = new command.remote.Command("join-k3s-master-3", { connection: conn(master3Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`, + triggers: [master3VmId], }, { dependsOn: [waitMaster3Ssh, joinMaster2] }); // --------------------------------------------------------------------------- @@ -123,22 +136,26 @@ const joinMaster3 = new command.remote.Command("join-k3s-master-3", { const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [worker1VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", { create: `for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [worker2VmId], interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); const joinWorker1 = new command.remote.Command("join-k3s-worker-1", { connection: conn(worker1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`, + triggers: [worker1VmId], }, { dependsOn: [waitWorker1Ssh] }); const joinWorker2 = new command.remote.Command("join-k3s-worker-2", { connection: conn(worker2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`, + triggers: [worker2VmId], }, { dependsOn: [waitWorker2Ssh] }); // --------------------------------------------------------------------------- @@ -148,6 +165,7 @@ const joinWorker2 = new command.remote.Command("join-k3s-worker-2", { const getKubeconfig = new command.remote.Command("get-kubeconfig", { connection: conn(master1Ip), create: `cat /etc/rancher/k3s/k3s.yaml`, + triggers: [master1VmId], }, { dependsOn: [joinWorker1, joinWorker2] }); export const kubeconfig = pulumi.secret(