kasun
afac1f338f
Every join and the exported kubeconfig previously pointed at master1Ip, so losing master-1 cut off API access even with etcd quorum intact. Use controlPlaneVipAddress (a pfSense IP Alias fronted by HAProxy across all three masters) instead, add it as a TLS SAN on master-1, and wait for the VIP's /healthz before any other node joins. The VIP is set up manually on pfSense rather than via a second kube-vip/ARP VIP: API access is too critical to depend on the same failover mechanism that serves LoadBalancer Services. Also strips trailing whitespace from the Longhorn disk mount script.
378 lines
14 KiB
TypeScript
378 lines
14 KiB
TypeScript
import * as pulumi from "@pulumi/pulumi";
|
|
import * as command from "@pulumi/command";
|
|
|
|
const config = new pulumi.Config();
|
|
|
|
const infraRef = new pulumi.StackReference(
|
|
`${pulumi.getOrganization()}/proxmox-infra/dev`,
|
|
);
|
|
|
|
const pve1Endpoint = infraRef.requireOutput(
|
|
"pve1Endpoint",
|
|
) as pulumi.Output<string>;
|
|
const pve1ApiToken = infraRef.requireOutput(
|
|
"pve1ApiToken",
|
|
) as pulumi.Output<string>;
|
|
const pve2Endpoint = infraRef.requireOutput(
|
|
"pve2Endpoint",
|
|
) as pulumi.Output<string>;
|
|
const pve2ApiToken = infraRef.requireOutput(
|
|
"pve2ApiToken",
|
|
) as pulumi.Output<string>;
|
|
const pve3Endpoint = infraRef.requireOutput(
|
|
"pve3Endpoint",
|
|
) as pulumi.Output<string>;
|
|
const pve3ApiToken = infraRef.requireOutput(
|
|
"pve3ApiToken",
|
|
) as pulumi.Output<string>;
|
|
|
|
// Node IPs — static DHCP leases set in the router
|
|
const master1Ip = infraRef.requireOutput("master1Ip");
|
|
const master2Ip = infraRef.requireOutput("master2Ip");
|
|
const master3Ip = infraRef.requireOutput("master3Ip");
|
|
const worker1Ip = infraRef.requireOutput("worker1Ip");
|
|
const worker2Ip = infraRef.requireOutput("worker2Ip");
|
|
const worker3Ip = infraRef.requireOutput("worker3Ip");
|
|
|
|
// Pre-shared k3s cluster token
|
|
const k3sToken = config.requireSecret("k3sToken");
|
|
|
|
// Control-plane VIP — floating IP for API server access, ARP-announced by
|
|
// kube-vip on whichever master is alive. Added as a TLS SAN on master-1's
|
|
// cert below, then used (instead of master1Ip) for every other node's join
|
|
// URL and for the exported kubeconfig, so the cluster stays reachable
|
|
// through the loss of any single master.
|
|
const controlPlaneVipAddress = config.requireSecret("controlPlaneVipAddress");
|
|
|
|
// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
|
|
const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<
|
|
Record<string, number>
|
|
>;
|
|
const ciRunnerPrivateKey = infraRef.requireOutput(
|
|
"ciRunnerPrivateKey",
|
|
) as pulumi.Output<string>;
|
|
|
|
const master1VmId = vmIdsOutput.apply((ids) => String(ids.master1));
|
|
const master2VmId = vmIdsOutput.apply((ids) => String(ids.master2));
|
|
const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3));
|
|
const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1));
|
|
const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2));
|
|
const worker3VmId = vmIdsOutput.apply((ids) => String(ids.worker3));
|
|
|
|
// SSH connection helper
|
|
function conn(
|
|
ip: pulumi.Input<string>,
|
|
): command.types.input.remote.ConnectionArgs {
|
|
return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra)
|
|
// triggers: vmId — forces re-run when VMs are deleted and recreated
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const startMaster1 = new command.local.Command("start-master-1", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [master1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startMaster2 = new command.local.Command("start-master-2", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [master2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startWorker1 = new command.local.Command("start-worker-1", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [worker1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startMaster3 = new command.local.Command("start-master-3", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [master3VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startWorker2 = new command.local.Command("start-worker-2", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [worker2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startWorker3 = new command.local.Command("start-worker-3", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve3ApiToken}" "${pve3Endpoint}/api2/json/nodes/pve/qemu/${worker3VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [worker3VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
|
|
const allStarts = [
|
|
startMaster1,
|
|
startMaster2,
|
|
startMaster3,
|
|
startWorker1,
|
|
startWorker2,
|
|
startWorker3,
|
|
];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 2: Wait for SSH on master-1, install k3s
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const waitMaster1Ssh = new command.local.Command(
|
|
"wait-ssh-master-1",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [master1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: allStarts },
|
|
);
|
|
|
|
const installMaster1 = new command.remote.Command(
|
|
"install-k3s-master-1",
|
|
{
|
|
connection: conn(master1Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --tls-san ${controlPlaneVipAddress} --disable servicelb --node-name k3s-master-1`,
|
|
triggers: [master1VmId],
|
|
},
|
|
{ dependsOn: [waitMaster1Ssh] },
|
|
);
|
|
|
|
const waitK3sMaster1Ready = new command.remote.Command(
|
|
"wait-k3s-master-1-ready",
|
|
{
|
|
connection: conn(master1Ip),
|
|
create: `for i in $(seq 1 60); do sudo kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`,
|
|
triggers: [master1VmId],
|
|
},
|
|
{ dependsOn: [installMaster1] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 3: Confirm the control-plane VIP is actually reachable before anyone
|
|
// else joins, so master-2/master-3 and every worker join through the VIP
|
|
// instead of master1Ip directly. The VIP itself is NOT managed here (or by
|
|
// any Pulumi stack) — it's a pfSense IP Alias Virtual IP with HAProxy
|
|
// TCP-load-balancing 6443 across all three masters, set up manually (see
|
|
// "pfSense one-time setup" in this stack's CLAUDE.md). This intentionally
|
|
// avoids a second kube-vip/ARP-based VIP for something as critical as API
|
|
// access — 03-k8s-infra's existing kube-vip DaemonSet already handles
|
|
// Service/LoadBalancer VIPs (e.g. Traefik) separately, entirely inside the
|
|
// cluster.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const waitControlPlaneVipReady = new command.remote.Command(
|
|
"wait-control-plane-vip-ready",
|
|
{
|
|
connection: conn(master1Ip),
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do curl -sk --max-time 3 https://${controlPlaneVipAddress}:6443/healthz && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [master1VmId],
|
|
},
|
|
{ dependsOn: [waitK3sMaster1Ready] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 4: Wait for SSH on remaining masters, join cluster via the VIP
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const waitMaster2Ssh = new command.local.Command(
|
|
"wait-ssh-master-2",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [master2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [waitK3sMaster1Ready] },
|
|
);
|
|
|
|
const waitMaster3Ssh = new command.local.Command(
|
|
"wait-ssh-master-3",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [master3VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [waitK3sMaster1Ready] },
|
|
);
|
|
|
|
const joinMaster2 = new command.remote.Command(
|
|
"join-k3s-master-2",
|
|
{
|
|
connection: conn(master2Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-2`,
|
|
triggers: [master2VmId],
|
|
},
|
|
{ dependsOn: [waitMaster2Ssh, waitControlPlaneVipReady] },
|
|
);
|
|
|
|
const joinMaster3 = new command.remote.Command(
|
|
"join-k3s-master-3",
|
|
{
|
|
connection: conn(master3Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-3`,
|
|
triggers: [master3VmId],
|
|
},
|
|
{ dependsOn: [waitMaster3Ssh, waitControlPlaneVipReady, joinMaster2] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 5: Join worker nodes via the VIP
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const waitWorker1Ssh = new command.local.Command(
|
|
"wait-ssh-worker-1",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [worker1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [joinMaster3] },
|
|
);
|
|
|
|
const waitWorker2Ssh = new command.local.Command(
|
|
"wait-ssh-worker-2",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [worker2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [joinMaster3] },
|
|
);
|
|
|
|
const waitWorker3Ssh = new command.local.Command(
|
|
"wait-ssh-worker-3",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [worker3VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [joinMaster3] },
|
|
);
|
|
|
|
const joinWorker1 = new command.remote.Command(
|
|
"join-k3s-worker-1",
|
|
{
|
|
connection: conn(worker1Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
|
|
triggers: [worker1VmId],
|
|
},
|
|
{ dependsOn: [waitWorker1Ssh] },
|
|
);
|
|
|
|
const joinWorker2 = new command.remote.Command(
|
|
"join-k3s-worker-2",
|
|
{
|
|
connection: conn(worker2Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
|
|
triggers: [worker2VmId],
|
|
},
|
|
{ dependsOn: [waitWorker2Ssh] },
|
|
);
|
|
|
|
const joinWorker3 = new command.remote.Command(
|
|
"join-k3s-worker-3",
|
|
{
|
|
connection: conn(worker3Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`,
|
|
triggers: [worker3VmId],
|
|
},
|
|
{ dependsOn: [waitWorker3Ssh] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 6: Format and mount Longhorn disks
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const workers = [worker1Ip, worker2Ip, worker3Ip];
|
|
const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh, waitWorker3Ssh];
|
|
|
|
workers.forEach((worker, i) => {
|
|
return new command.remote.Command(
|
|
`mount-longhorn-disk-${i}`,
|
|
{
|
|
connection: conn(worker),
|
|
create: `set -euo pipefail;
|
|
DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1');
|
|
DISK_PATH="/dev/disk/by-id/$DISK_ID";
|
|
sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH";
|
|
sudo mkdir -p /mnt/longhorn-extra;
|
|
echo ""$DISK_PATH" /mnt/longhorn-extra ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab;
|
|
sudo mount -a;
|
|
df -h /mnt/longhorn-extra`,
|
|
},
|
|
{ dependsOn: workerSshReader[i] },
|
|
);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 7: Read kubeconfig from master-1, patch server URL to the VIP
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const getKubeconfig = new command.remote.Command(
|
|
"get-kubeconfig",
|
|
{
|
|
connection: conn(master1Ip),
|
|
create: `sudo cat /etc/rancher/k3s/k3s.yaml`,
|
|
triggers: [master1VmId],
|
|
},
|
|
{ dependsOn: [joinWorker1, joinWorker2, joinWorker3] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 8: Install additional packages
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const allVms = [
|
|
{
|
|
id: master1VmId,
|
|
ip: master1Ip,
|
|
name: "master-1",
|
|
readiness: waitMaster1Ssh,
|
|
},
|
|
{
|
|
id: master2VmId,
|
|
ip: master2Ip,
|
|
name: "master-2",
|
|
readiness: waitMaster2Ssh,
|
|
},
|
|
{
|
|
id: master3VmId,
|
|
ip: master3Ip,
|
|
name: "master-3",
|
|
readiness: waitMaster3Ssh,
|
|
},
|
|
{
|
|
id: worker1VmId,
|
|
ip: worker1Ip,
|
|
name: "worker-1",
|
|
readiness: waitWorker1Ssh,
|
|
},
|
|
{
|
|
id: worker2VmId,
|
|
ip: worker2Ip,
|
|
name: "worker-2",
|
|
readiness: waitWorker2Ssh,
|
|
},
|
|
{
|
|
id: worker3VmId,
|
|
ip: worker3Ip,
|
|
name: "worker-3",
|
|
readiness: waitWorker3Ssh,
|
|
},
|
|
];
|
|
|
|
allVms.forEach(({ id, ip, name, readiness }) => {
|
|
new command.remote.Command(
|
|
`install-packages-${name}`,
|
|
{
|
|
connection: conn(ip),
|
|
create: pulumi.interpolate`sudo apt update && sudo apt install -y qemu-guest-agent && sudo systemctl enable --now qemu-guest-agent`,
|
|
triggers: [id],
|
|
},
|
|
{ dependsOn: readiness },
|
|
);
|
|
});
|
|
|
|
export const kubeconfig = pulumi.secret(
|
|
pulumi
|
|
.all([getKubeconfig.stdout, controlPlaneVipAddress])
|
|
.apply(([kc, vip]) => kc.replaceAll("127.0.0.1", vip as string).trim()),
|
|
);
|