From afac1f338fffa42b41e43ef7ac9032c3d06895fb Mon Sep 17 00:00:00 2001 From: kasun Date: Fri, 2 Oct 2026 23:52:32 +0200 Subject: [PATCH] feature(k8s-bootstrap): join nodes and export kubeconfig via control-plane VIP Every join and the exported kubeconfig previously pointed at master1Ip, so losing master-1 cut off API access even with etcd quorum intact. Use controlPlaneVipAddress (a pfSense IP Alias fronted by HAProxy across all three masters) instead, add it as a TLS SAN on master-1, and wait for the VIP's /healthz before any other node joins. The VIP is set up manually on pfSense rather than via a second kube-vip/ARP VIP: API access is too critical to depend on the same failover mechanism that serves LoadBalancer Services. Also strips trailing whitespace from the Longhorn disk mount script. --- 02-k8s-bootstrap/index.ts | 74 +++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/02-k8s-bootstrap/index.ts b/02-k8s-bootstrap/index.ts index 2f7264f..da96eda 100644 --- a/02-k8s-bootstrap/index.ts +++ b/02-k8s-bootstrap/index.ts @@ -37,6 +37,13 @@ const worker3Ip = infraRef.requireOutput("worker3Ip"); // Pre-shared k3s cluster token const k3sToken = config.requireSecret("k3sToken"); +// Control-plane VIP — floating IP for API server access, ARP-announced by +// kube-vip on whichever master is alive. Added as a TLS SAN on master-1's +// cert below, then used (instead of master1Ip) for every other node's join +// URL and for the exported kubeconfig, so the cluster stays reachable +// through the loss of any single master. +const controlPlaneVipAddress = config.requireSecret("controlPlaneVipAddress"); + // VM IDs and CI runner SSH key — read from proxmox-infra stack outputs const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output< Record @@ -122,7 +129,7 @@ const installMaster1 = new command.remote.Command( "install-k3s-master-1", { connection: conn(master1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --disable servicelb --node-name k3s-master-1`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --tls-san ${controlPlaneVipAddress} --disable servicelb --node-name k3s-master-1`, triggers: [master1VmId], }, { dependsOn: [waitMaster1Ssh] }, @@ -139,7 +146,30 @@ const waitK3sMaster1Ready = new command.remote.Command( ); // --------------------------------------------------------------------------- -// Step 3: Wait for SSH on remaining masters, join cluster +// Step 3: Confirm the control-plane VIP is actually reachable before anyone +// else joins, so master-2/master-3 and every worker join through the VIP +// instead of master1Ip directly. The VIP itself is NOT managed here (or by +// any Pulumi stack) — it's a pfSense IP Alias Virtual IP with HAProxy +// TCP-load-balancing 6443 across all three masters, set up manually (see +// "pfSense one-time setup" in this stack's CLAUDE.md). This intentionally +// avoids a second kube-vip/ARP-based VIP for something as critical as API +// access — 03-k8s-infra's existing kube-vip DaemonSet already handles +// Service/LoadBalancer VIPs (e.g. Traefik) separately, entirely inside the +// cluster. +// --------------------------------------------------------------------------- + +const waitControlPlaneVipReady = new command.remote.Command( + "wait-control-plane-vip-ready", + { + connection: conn(master1Ip), + create: pulumi.interpolate`for i in $(seq 1 60); do curl -sk --max-time 3 https://${controlPlaneVipAddress}:6443/healthz && exit 0; sleep 5; done; exit 1`, + triggers: [master1VmId], + }, + { dependsOn: [waitK3sMaster1Ready] }, +); + +// --------------------------------------------------------------------------- +// Step 4: Wait for SSH on remaining masters, join cluster via the VIP // --------------------------------------------------------------------------- const waitMaster2Ssh = new command.local.Command( @@ -166,24 +196,24 @@ const joinMaster2 = new command.remote.Command( "join-k3s-master-2", { connection: conn(master2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-2`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-2`, triggers: [master2VmId], }, - { dependsOn: [waitMaster2Ssh] }, + { dependsOn: [waitMaster2Ssh, waitControlPlaneVipReady] }, ); const joinMaster3 = new command.remote.Command( "join-k3s-master-3", { connection: conn(master3Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-3`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-3`, triggers: [master3VmId], }, - { dependsOn: [waitMaster3Ssh, joinMaster2] }, + { dependsOn: [waitMaster3Ssh, waitControlPlaneVipReady, joinMaster2] }, ); // --------------------------------------------------------------------------- -// Step 4: Join worker nodes +// Step 5: Join worker nodes via the VIP // --------------------------------------------------------------------------- const waitWorker1Ssh = new command.local.Command( @@ -220,7 +250,7 @@ const joinWorker1 = new command.remote.Command( "join-k3s-worker-1", { connection: conn(worker1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`, triggers: [worker1VmId], }, { dependsOn: [waitWorker1Ssh] }, @@ -230,7 +260,7 @@ const joinWorker2 = new command.remote.Command( "join-k3s-worker-2", { connection: conn(worker2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`, triggers: [worker2VmId], }, { dependsOn: [waitWorker2Ssh] }, @@ -240,14 +270,14 @@ const joinWorker3 = new command.remote.Command( "join-k3s-worker-3", { connection: conn(worker3Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`, triggers: [worker3VmId], }, { dependsOn: [waitWorker3Ssh] }, ); // --------------------------------------------------------------------------- -// Step 5: Format and mount Longhorn disks +// Step 6: Format and mount Longhorn disks // --------------------------------------------------------------------------- const workers = [worker1Ip, worker2Ip, worker3Ip]; @@ -258,13 +288,13 @@ workers.forEach((worker, i) => { `mount-longhorn-disk-${i}`, { connection: conn(worker), - create: `set -euo pipefail; - DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1'); - DISK_PATH="/dev/disk/by-id/$DISK_ID"; - sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH"; - sudo mkdir -p /mnt/longhorn-extra; - echo ""$DISK_PATH" /mnt/longhorn-extra ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab; - sudo mount -a; + create: `set -euo pipefail; + DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1'); + DISK_PATH="/dev/disk/by-id/$DISK_ID"; + sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH"; + sudo mkdir -p /mnt/longhorn-extra; + echo ""$DISK_PATH" /mnt/longhorn-extra ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab; + sudo mount -a; df -h /mnt/longhorn-extra`, }, { dependsOn: workerSshReader[i] }, @@ -272,7 +302,7 @@ workers.forEach((worker, i) => { }); // --------------------------------------------------------------------------- -// Step 6: Read kubeconfig from master-1, patch server URL for external access +// Step 7: Read kubeconfig from master-1, patch server URL to the VIP // --------------------------------------------------------------------------- const getKubeconfig = new command.remote.Command( @@ -286,7 +316,7 @@ const getKubeconfig = new command.remote.Command( ); // --------------------------------------------------------------------------- -// Step 7: Install additional packages +// Step 8: Install additional packages // --------------------------------------------------------------------------- const allVms = [ @@ -342,6 +372,6 @@ allVms.forEach(({ id, ip, name, readiness }) => { export const kubeconfig = pulumi.secret( pulumi - .all([getKubeconfig.stdout, master1Ip]) - .apply(([kc, ip]) => kc.replaceAll("127.0.0.1", ip as string).trim()), + .all([getKubeconfig.stdout, controlPlaneVipAddress]) + .apply(([kc, vip]) => kc.replaceAll("127.0.0.1", vip as string).trim()), );