From 3f9887400825c3d98b72a15251cc7b861f43ba67 Mon Sep 17 00:00:00 2001 From: kasun Date: Tue, 28 Jul 2026 22:52:22 +0200 Subject: [PATCH] feature: join k3s-worker-3 into the cluster Wires the k8s-bootstrap join sequence (start, SSH wait, k3s agent join, Longhorn disk mount) and the k8s-infra Longhorn disk patch for worker-3. Its VM was already provisioned by proxmox-infra in an earlier branch. --- 02-k8s-bootstrap/CLAUDE.md | 5 +++-- 02-k8s-bootstrap/index.ts | 40 +++++++++++++++++++++++++++++++++++--- 03-k8s-infra/index.ts | 2 +- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/02-k8s-bootstrap/CLAUDE.md b/02-k8s-bootstrap/CLAUDE.md index fc7d1cf..4160524 100644 --- a/02-k8s-bootstrap/CLAUDE.md +++ b/02-k8s-bootstrap/CLAUDE.md @@ -10,7 +10,7 @@ Bootstraps a k3s cluster on the Proxmox VMs created by `proxmox-infra`. Starts V 2. Waits for port 22 to open on each VM (bash `/dev/tcp`) 3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san ` 4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes -5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes +5. Joins `k3s-worker-1`, `k3s-worker-2`, and `k3s-worker-3` as agent nodes 6. Formats the workers' dedicated `scsi1` disk (ext4, if not already formatted) and mounts it at `/mnt/longhorn-extra` (via `/etc/fstab`) — this is the disk `03-k8s-infra` later registers as an extra Longhorn disk, to keep Longhorn off the root filesystem and avoid `DiskPressure` 7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig` @@ -28,9 +28,10 @@ pulumi config set master2Ip "192.168.1.x" pulumi config set master3Ip "192.168.1.x" pulumi config set worker1Ip "192.168.1.x" pulumi config set worker2Ip "192.168.1.x" +pulumi config set worker3Ip "192.168.1.x" ``` -Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`) are read automatically from `proxmox-infra` via StackReference — do **not** set them here. +Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`, `pve3Endpoint`, `pve3ApiToken`) are read automatically from `proxmox-infra` via StackReference — do **not** set them here. ## Prerequisites diff --git a/02-k8s-bootstrap/index.ts b/02-k8s-bootstrap/index.ts index 5732777..c839745 100644 --- a/02-k8s-bootstrap/index.ts +++ b/02-k8s-bootstrap/index.ts @@ -21,6 +21,12 @@ const pve2Endpoint = infraRef.requireOutput( const pve2ApiToken = infraRef.requireOutput( "pve2ApiToken", ) as pulumi.Output; +const pve3Endpoint = infraRef.requireOutput( + "pve3Endpoint", +) as pulumi.Output; +const pve3ApiToken = infraRef.requireOutput( + "pve3ApiToken", +) as pulumi.Output; // Node IPs — static DHCP leases set in the router const master1Ip = infraRef.requireOutput("master1Ip"); @@ -28,6 +34,7 @@ const master2Ip = infraRef.requireOutput("master2Ip"); const master3Ip = infraRef.requireOutput("master3Ip"); const worker1Ip = infraRef.requireOutput("worker1Ip"); const worker2Ip = infraRef.requireOutput("worker2Ip"); +const worker3Ip = infraRef.requireOutput("worker3Ip"); // Pre-shared k3s cluster token const k3sToken = config.requireSecret("k3sToken"); @@ -45,6 +52,7 @@ const master2VmId = vmIdsOutput.apply((ids) => String(ids.master2)); const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3)); const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1)); const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2)); +const worker3VmId = vmIdsOutput.apply((ids) => String(ids.worker3)); // SSH connection helper function conn( @@ -83,6 +91,11 @@ const startWorker2 = new command.local.Command("start-worker-2", { triggers: [worker2VmId], interpreter: ["/bin/bash", "-c"], }); +const startWorker3 = new command.local.Command("start-worker-3", { + create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve3ApiToken}" "${pve3Endpoint}/api2/json/nodes/pve/qemu/${worker3VmId}/status/start" 2>/dev/null || true`, + triggers: [worker3VmId], + interpreter: ["/bin/bash", "-c"], +}); const allStarts = [ startMaster1, @@ -90,6 +103,7 @@ const allStarts = [ startMaster3, startWorker1, startWorker2, + startWorker3, ]; // --------------------------------------------------------------------------- @@ -194,6 +208,16 @@ const waitWorker2Ssh = new command.local.Command( { dependsOn: [joinMaster3] }, ); +const waitWorker3Ssh = new command.local.Command( + "wait-ssh-worker-3", + { + create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`, + triggers: [worker3VmId], + interpreter: ["/bin/bash", "-c"], + }, + { dependsOn: [joinMaster3] }, +); + const joinWorker1 = new command.remote.Command( "join-k3s-worker-1", { @@ -214,12 +238,22 @@ const joinWorker2 = new command.remote.Command( { dependsOn: [waitWorker2Ssh] }, ); +const joinWorker3 = new command.remote.Command( + "join-k3s-worker-3", + { + connection: conn(worker3Ip), + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`, + triggers: [worker3VmId], + }, + { dependsOn: [waitWorker3Ssh] }, +); + // --------------------------------------------------------------------------- // Step 5: Format and mount Longhorn disks // --------------------------------------------------------------------------- -const workers = [worker1Ip, worker2Ip]; -const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh]; +const workers = [worker1Ip, worker2Ip, worker3Ip]; +const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh, waitWorker3Ssh]; const mountLonghornDisks = workers.map((worker, i) => { return new command.remote.Command( @@ -250,7 +284,7 @@ const getKubeconfig = new command.remote.Command( create: `sudo cat /etc/rancher/k3s/k3s.yaml`, triggers: [master1VmId], }, - { dependsOn: [joinWorker1, joinWorker2] }, + { dependsOn: [joinWorker1, joinWorker2, joinWorker3] }, ); export const kubeconfig = pulumi.secret( diff --git a/03-k8s-infra/index.ts b/03-k8s-infra/index.ts index cc901ab..947c9f0 100644 --- a/03-k8s-infra/index.ts +++ b/03-k8s-infra/index.ts @@ -143,7 +143,7 @@ const longhorn = new k8s.helm.v3.Release( ); // Patches Longhorn instances to use the mountend Longhorn storage on the worker nodes. -["k3s-worker-1", "k3s-worker-2"].forEach( +["k3s-worker-1", "k3s-worker-2", "k3s-worker-3"].forEach( (nodeName) => new k8s.apiextensions.CustomResourcePatch( `${nodeName}-longhorn-disks`,