homelab-infrastructure-as-code/k8s-bootstrap/index.ts

import * as pulumi from "@pulumi/pulumi";
import * as command from "@pulumi/command";

const config = new pulumi.Config();

//fetch credentials from proxmox-infra
const infraRef = new pulumi.StackReference(
  `${pulumi.getOrganization()}/proxmox-infra/dev`,
);

// Proxmox API credentials — same as proxmox-infra stack
const pve1Endpoint = infraRef.requireOutput(
  "pve1Endpoint",
) as pulumi.Output<string>;
const pve1ApiToken = infraRef.requireOutput(
  "pve1ApiToken",
) as pulumi.Output<string>;
const pve2Endpoint = infraRef.requireOutput(
  "pve2Endpoint",
) as pulumi.Output<string>;
const pve2ApiToken = infraRef.requireOutput(
  "pve2ApiToken",
) as pulumi.Output<string>;

// Node IPs — static DHCP leases set in the router
const master1Ip = config.require("master1Ip");
const master2Ip = config.require("master2Ip");
const master3Ip = config.require("master3Ip");
const worker1Ip = config.require("worker1Ip");
const worker2Ip = config.require("worker2Ip");

// Pre-shared k3s cluster token
const k3sToken = config.requireSecret("k3sToken");

// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<Record<string, number>>;
const ciRunnerPrivateKey = infraRef.requireOutput("ciRunnerPrivateKey") as pulumi.Output<string>;

const master1VmId = vmIdsOutput.apply(ids => String(ids.master1));
const master2VmId = vmIdsOutput.apply(ids => String(ids.master2));
const master3VmId = vmIdsOutput.apply(ids => String(ids.master3));
const worker1VmId = vmIdsOutput.apply(ids => String(ids.worker1));
const worker2VmId = vmIdsOutput.apply(ids => String(ids.worker2));

// SSH connection helper
function conn(ip: string): command.types.input.remote.ConnectionArgs {
    return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
}

// ---------------------------------------------------------------------------
// Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra)
// ---------------------------------------------------------------------------

const startMaster1 = new command.local.Command("start-master-1", {
    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`,
    interpreter: ["/bin/bash", "-c"],
});
const startMaster2 = new command.local.Command("start-master-2", {
    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`,
    interpreter: ["/bin/bash", "-c"],
});
const startWorker1 = new command.local.Command("start-worker-1", {
    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`,
    interpreter: ["/bin/bash", "-c"],
});
const startMaster3 = new command.local.Command("start-master-3", {
    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`,
    interpreter: ["/bin/bash", "-c"],
});
const startWorker2 = new command.local.Command("start-worker-2", {
    create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`,
    interpreter: ["/bin/bash", "-c"],
});

const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, startWorker2];

// ---------------------------------------------------------------------------
// Step 2: Wait for SSH on master-1, install k3s
// ---------------------------------------------------------------------------

const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", {
    create: `for i in $(seq 1 60); do nc -z -w 5 ${master1Ip} 22 && exit 0; sleep 5; done; exit 1`,
    interpreter: ["/bin/bash", "-c"],
}, { dependsOn: allStarts });

const installMaster1 = new command.remote.Command("install-k3s-master-1", {
    connection: conn(master1Ip),
    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip}`,
}, { dependsOn: [waitMaster1Ssh] });

const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", {
    connection: conn(master1Ip),
    create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`,
}, { dependsOn: [installMaster1] });

// ---------------------------------------------------------------------------
// Step 3: Wait for SSH on remaining masters, join cluster
// ---------------------------------------------------------------------------

const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", {
    create: `for i in $(seq 1 60); do nc -z -w 5 ${master2Ip} 22 && exit 0; sleep 5; done; exit 1`,
    interpreter: ["/bin/bash", "-c"],
}, { dependsOn: [waitK3sMaster1Ready] });

const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", {
    create: `for i in $(seq 1 60); do nc -z -w 5 ${master3Ip} 22 && exit 0; sleep 5; done; exit 1`,
    interpreter: ["/bin/bash", "-c"],
}, { dependsOn: [waitK3sMaster1Ready] });

const joinMaster2 = new command.remote.Command("join-k3s-master-2", {
    connection: conn(master2Ip),
    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
}, { dependsOn: [waitMaster2Ssh] });

const joinMaster3 = new command.remote.Command("join-k3s-master-3", {
    connection: conn(master3Ip),
    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
}, { dependsOn: [waitMaster3Ssh, joinMaster2] });

// ---------------------------------------------------------------------------
// Step 4: Join worker nodes
// ---------------------------------------------------------------------------

const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", {
    create: `for i in $(seq 1 60); do nc -z -w 5 ${worker1Ip} 22 && exit 0; sleep 5; done; exit 1`,
    interpreter: ["/bin/bash", "-c"],
}, { dependsOn: [joinMaster3] });

const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", {
    create: `for i in $(seq 1 60); do nc -z -w 5 ${worker2Ip} 22 && exit 0; sleep 5; done; exit 1`,
    interpreter: ["/bin/bash", "-c"],
}, { dependsOn: [joinMaster3] });

const joinWorker1 = new command.remote.Command("join-k3s-worker-1", {
    connection: conn(worker1Ip),
    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
}, { dependsOn: [waitWorker1Ssh] });

const joinWorker2 = new command.remote.Command("join-k3s-worker-2", {
    connection: conn(worker2Ip),
    create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
}, { dependsOn: [waitWorker2Ssh] });

// ---------------------------------------------------------------------------
// Step 5: Read kubeconfig from master-1, patch server URL for external access
// ---------------------------------------------------------------------------

const getKubeconfig = new command.remote.Command("get-kubeconfig", {
    connection: conn(master1Ip),
    create: `cat /etc/rancher/k3s/k3s.yaml`,
}, { dependsOn: [joinWorker1, joinWorker2] });

export const kubeconfig = pulumi.secret(
    getKubeconfig.stdout.apply(kc => kc.replace(/127\.0\.0\.1/g, master1Ip).trim()),
);