236 lines
8.9 KiB
TypeScript
236 lines
8.9 KiB
TypeScript
import * as pulumi from "@pulumi/pulumi";
|
|
import * as command from "@pulumi/command";
|
|
|
|
const config = new pulumi.Config();
|
|
|
|
//fetch credentials from proxmox-infra
|
|
const infraRef = new pulumi.StackReference(
|
|
`${pulumi.getOrganization()}/proxmox-infra/dev`,
|
|
);
|
|
|
|
// Proxmox API credentials — same as proxmox-infra stack
|
|
const pve1Endpoint = infraRef.requireOutput(
|
|
"pve1Endpoint",
|
|
) as pulumi.Output<string>;
|
|
const pve1ApiToken = infraRef.requireOutput(
|
|
"pve1ApiToken",
|
|
) as pulumi.Output<string>;
|
|
const pve2Endpoint = infraRef.requireOutput(
|
|
"pve2Endpoint",
|
|
) as pulumi.Output<string>;
|
|
const pve2ApiToken = infraRef.requireOutput(
|
|
"pve2ApiToken",
|
|
) as pulumi.Output<string>;
|
|
|
|
// Node IPs — static DHCP leases set in the router
|
|
const master1Ip = infraRef.requireOutput("master1Ip");
|
|
const master2Ip = infraRef.requireOutput("master2Ip");
|
|
const master3Ip = infraRef.requireOutput("master3Ip");
|
|
const worker1Ip = infraRef.requireOutput("worker1Ip");
|
|
const worker2Ip = infraRef.requireOutput("worker2Ip");
|
|
|
|
// Pre-shared k3s cluster token
|
|
const k3sToken = config.requireSecret("k3sToken");
|
|
|
|
// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
|
|
const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<
|
|
Record<string, number>
|
|
>;
|
|
const ciRunnerPrivateKey = infraRef.requireOutput(
|
|
"ciRunnerPrivateKey",
|
|
) as pulumi.Output<string>;
|
|
|
|
const master1VmId = vmIdsOutput.apply((ids) => String(ids.master1));
|
|
const master2VmId = vmIdsOutput.apply((ids) => String(ids.master2));
|
|
const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3));
|
|
const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1));
|
|
const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2));
|
|
|
|
// SSH connection helper
|
|
function conn(
|
|
ip: pulumi.Input<string>,
|
|
): command.types.input.remote.ConnectionArgs {
|
|
return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra)
|
|
// triggers: vmId — forces re-run when VMs are deleted and recreated
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const startMaster1 = new command.local.Command("start-master-1", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [master1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startMaster2 = new command.local.Command("start-master-2", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [master2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startWorker1 = new command.local.Command("start-worker-1", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [worker1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startMaster3 = new command.local.Command("start-master-3", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [master3VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
const startWorker2 = new command.local.Command("start-worker-2", {
|
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`,
|
|
triggers: [worker2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
});
|
|
|
|
const allStarts = [
|
|
startMaster1,
|
|
startMaster2,
|
|
startMaster3,
|
|
startWorker1,
|
|
startWorker2,
|
|
];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 2: Wait for SSH on master-1, install k3s
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const waitMaster1Ssh = new command.local.Command(
|
|
"wait-ssh-master-1",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [master1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: allStarts },
|
|
);
|
|
|
|
const installMaster1 = new command.remote.Command(
|
|
"install-k3s-master-1",
|
|
{
|
|
connection: conn(master1Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`,
|
|
triggers: [master1VmId],
|
|
},
|
|
{ dependsOn: [waitMaster1Ssh] },
|
|
);
|
|
|
|
const waitK3sMaster1Ready = new command.remote.Command(
|
|
"wait-k3s-master-1-ready",
|
|
{
|
|
connection: conn(master1Ip),
|
|
create: `for i in $(seq 1 60); do sudo kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`,
|
|
triggers: [master1VmId],
|
|
},
|
|
{ dependsOn: [installMaster1] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 3: Wait for SSH on remaining masters, join cluster
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const waitMaster2Ssh = new command.local.Command(
|
|
"wait-ssh-master-2",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [master2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [waitK3sMaster1Ready] },
|
|
);
|
|
|
|
const waitMaster3Ssh = new command.local.Command(
|
|
"wait-ssh-master-3",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${master3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [master3VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [waitK3sMaster1Ready] },
|
|
);
|
|
|
|
const joinMaster2 = new command.remote.Command(
|
|
"join-k3s-master-2",
|
|
{
|
|
connection: conn(master2Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`,
|
|
triggers: [master2VmId],
|
|
},
|
|
{ dependsOn: [waitMaster2Ssh] },
|
|
);
|
|
|
|
const joinMaster3 = new command.remote.Command(
|
|
"join-k3s-master-3",
|
|
{
|
|
connection: conn(master3Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`,
|
|
triggers: [master3VmId],
|
|
},
|
|
{ dependsOn: [waitMaster3Ssh, joinMaster2] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 4: Join worker nodes
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const waitWorker1Ssh = new command.local.Command(
|
|
"wait-ssh-worker-1",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker1Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [worker1VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [joinMaster3] },
|
|
);
|
|
|
|
const waitWorker2Ssh = new command.local.Command(
|
|
"wait-ssh-worker-2",
|
|
{
|
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker2Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
|
triggers: [worker2VmId],
|
|
interpreter: ["/bin/bash", "-c"],
|
|
},
|
|
{ dependsOn: [joinMaster3] },
|
|
);
|
|
|
|
const joinWorker1 = new command.remote.Command(
|
|
"join-k3s-worker-1",
|
|
{
|
|
connection: conn(worker1Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
|
|
triggers: [worker1VmId],
|
|
},
|
|
{ dependsOn: [waitWorker1Ssh] },
|
|
);
|
|
|
|
const joinWorker2 = new command.remote.Command(
|
|
"join-k3s-worker-2",
|
|
{
|
|
connection: conn(worker2Ip),
|
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
|
|
triggers: [worker2VmId],
|
|
},
|
|
{ dependsOn: [waitWorker2Ssh] },
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Step 5: Read kubeconfig from master-1, patch server URL for external access
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const getKubeconfig = new command.remote.Command(
|
|
"get-kubeconfig",
|
|
{
|
|
connection: conn(master1Ip),
|
|
create: `sudo cat /etc/rancher/k3s/k3s.yaml`,
|
|
triggers: [master1VmId],
|
|
},
|
|
{ dependsOn: [joinWorker1, joinWorker2] },
|
|
);
|
|
|
|
export const kubeconfig = pulumi.secret(
|
|
pulumi
|
|
.all([getKubeconfig.stdout, master1Ip])
|
|
.apply(([kc, ip]) => kc.replaceAll("127.0.0.1", ip as string).trim()),
|
|
);
|