import * as pulumi from "@pulumi/pulumi"; import * as command from "@pulumi/command"; const config = new pulumi.Config(); // Proxmox API credentials — same as proxmox-infra stack const pve1Endpoint = config.requireSecret("pve1Endpoint"); const pve1ApiToken = config.requireSecret("pve1ApiToken"); const pve2Endpoint = config.requireSecret("pve2Endpoint"); const pve2ApiToken = config.requireSecret("pve2ApiToken"); // Node IPs — static DHCP leases set in the router const master1Ip = config.require("master1Ip"); const master2Ip = config.require("master2Ip"); const master3Ip = config.require("master3Ip"); const worker1Ip = config.require("worker1Ip"); const worker2Ip = config.require("worker2Ip"); // Pre-shared k3s cluster token const k3sToken = config.requireSecret("k3sToken"); // VM IDs and CI runner SSH key — read from proxmox-infra stack outputs const infraRef = new pulumi.StackReference(`${pulumi.getOrganization()}/proxmox-infra/dev`); const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output>; const ciRunnerPrivateKey = infraRef.requireOutput("ciRunnerPrivateKey") as pulumi.Output; const master1VmId = vmIdsOutput.apply(ids => String(ids.master1)); const master2VmId = vmIdsOutput.apply(ids => String(ids.master2)); const master3VmId = vmIdsOutput.apply(ids => String(ids.master3)); const worker1VmId = vmIdsOutput.apply(ids => String(ids.worker1)); const worker2VmId = vmIdsOutput.apply(ids => String(ids.worker2)); // SSH connection helper function conn(ip: string): command.types.input.remote.ConnectionArgs { return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey }; } // --------------------------------------------------------------------------- // Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra) // --------------------------------------------------------------------------- const startMaster1 = new command.local.Command("start-master-1", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`, interpreter: ["/bin/bash", "-c"], }); const startMaster2 = new command.local.Command("start-master-2", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`, interpreter: ["/bin/bash", "-c"], }); const startWorker1 = new command.local.Command("start-worker-1", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`, interpreter: ["/bin/bash", "-c"], }); const startMaster3 = new command.local.Command("start-master-3", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`, interpreter: ["/bin/bash", "-c"], }); const startWorker2 = new command.local.Command("start-worker-2", { create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`, interpreter: ["/bin/bash", "-c"], }); const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, startWorker2]; // --------------------------------------------------------------------------- // Step 2: Wait for SSH on master-1, install k3s // --------------------------------------------------------------------------- const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", { create: `for i in $(seq 1 60); do nc -z -w 5 ${master1Ip} 22 && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: allStarts }); const installMaster1 = new command.remote.Command("install-k3s-master-1", { connection: conn(master1Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip}`, }, { dependsOn: [waitMaster1Ssh] }); const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", { connection: conn(master1Ip), create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`, }, { dependsOn: [installMaster1] }); // --------------------------------------------------------------------------- // Step 3: Wait for SSH on remaining masters, join cluster // --------------------------------------------------------------------------- const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", { create: `for i in $(seq 1 60); do nc -z -w 5 ${master2Ip} 22 && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", { create: `for i in $(seq 1 60); do nc -z -w 5 ${master3Ip} 22 && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [waitK3sMaster1Ready] }); const joinMaster2 = new command.remote.Command("join-k3s-master-2", { connection: conn(master2Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`, }, { dependsOn: [waitMaster2Ssh] }); const joinMaster3 = new command.remote.Command("join-k3s-master-3", { connection: conn(master3Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`, }, { dependsOn: [waitMaster3Ssh, joinMaster2] }); // --------------------------------------------------------------------------- // Step 4: Join worker nodes // --------------------------------------------------------------------------- const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", { create: `for i in $(seq 1 60); do nc -z -w 5 ${worker1Ip} 22 && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", { create: `for i in $(seq 1 60); do nc -z -w 5 ${worker2Ip} 22 && exit 0; sleep 5; done; exit 1`, interpreter: ["/bin/bash", "-c"], }, { dependsOn: [joinMaster3] }); const joinWorker1 = new command.remote.Command("join-k3s-worker-1", { connection: conn(worker1Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`, }, { dependsOn: [waitWorker1Ssh] }); const joinWorker2 = new command.remote.Command("join-k3s-worker-2", { connection: conn(worker2Ip), create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`, }, { dependsOn: [waitWorker2Ssh] }); // --------------------------------------------------------------------------- // Step 5: Read kubeconfig from master-1, patch server URL for external access // --------------------------------------------------------------------------- const getKubeconfig = new command.remote.Command("get-kubeconfig", { connection: conn(master1Ip), create: `cat /etc/rancher/k3s/k3s.yaml`, }, { dependsOn: [joinWorker1, joinWorker2] }); export const kubeconfig = pulumi.secret( getKubeconfig.stdout.apply(kc => kc.replace(/127\.0\.0\.1/g, master1Ip).trim()), );