kasun/homelab-infrastructure-as-code
1
0 Fork 0
Code Issues 11 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

feature(k8s-bootstrap): join nodes and export kubeconfig via control-plane VIP

Browse Source 
Every join and the exported kubeconfig previously pointed at master1Ip,
so losing master-1 cut off API access even with etcd quorum intact. Use
controlPlaneVipAddress (a pfSense IP Alias fronted by HAProxy across all
three masters) instead, add it as a TLS SAN on master-1, and wait for
the VIP's /healthz before any other node joins.

The VIP is set up manually on pfSense rather than via a second
kube-vip/ARP VIP: API access is too critical to depend on the same
failover mechanism that serves LoadBalancer Services.

Also strips trailing whitespace from the Longhorn disk mount script.
This commit is contained in:
kasun committed 2026-10-02 23:52:32 +02:00
1 parent 2f57f1822f
commit afac1f338f
1 file changed
+52 -22
Download Patch File Download Diff File Expand all files Collapse all files
02-k8s-bootstrap/index.ts
+52 -22
View File
@@ -37,6 +37,13 @@ const worker3Ip = infraRef.requireOutput("worker3Ip");
// Pre-shared k3s cluster token
const k3sToken = config.requireSecret("k3sToken");
// Control-plane VIP — floating IP for API server access, ARP-announced by
// kube-vip on whichever master is alive. Added as a TLS SAN on master-1's
// cert below, then used (instead of master1Ip) for every other node's join
// URL and for the exported kubeconfig, so the cluster stays reachable
// through the loss of any single master.
const controlPlaneVipAddress = config.requireSecret("controlPlaneVipAddress");
// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<
Record<string, number>
@@ -122,7 +129,7 @@ const installMaster1 = new command.remote.Command(
"install-k3s-master-1",
{
connection: conn(master1Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --disable servicelb --node-name k3s-master-1`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --tls-san ${controlPlaneVipAddress} --disable servicelb --node-name k3s-master-1`,
triggers: [master1VmId],
},
{ dependsOn: [waitMaster1Ssh] },
@@ -139,7 +146,30 @@ const waitK3sMaster1Ready = new command.remote.Command(
);
// ---------------------------------------------------------------------------
// Step 3: Wait for SSH on remaining masters, join cluster
// Step 3: Confirm the control-plane VIP is actually reachable before anyone
// else joins, so master-2/master-3 and every worker join through the VIP
// instead of master1Ip directly. The VIP itself is NOT managed here (or by
// any Pulumi stack) — it's a pfSense IP Alias Virtual IP with HAProxy
// TCP-load-balancing 6443 across all three masters, set up manually (see
// "pfSense one-time setup" in this stack's CLAUDE.md). This intentionally
// avoids a second kube-vip/ARP-based VIP for something as critical as API
// access — 03-k8s-infra's existing kube-vip DaemonSet already handles
// Service/LoadBalancer VIPs (e.g. Traefik) separately, entirely inside the
// cluster.
// ---------------------------------------------------------------------------
const waitControlPlaneVipReady = new command.remote.Command(
"wait-control-plane-vip-ready",
{
connection: conn(master1Ip),
create: pulumi.interpolate`for i in $(seq 1 60); do curl -sk --max-time 3 https://${controlPlaneVipAddress}:6443/healthz && exit 0; sleep 5; done; exit 1`,
triggers: [master1VmId],
},
{ dependsOn: [waitK3sMaster1Ready] },
);
// ---------------------------------------------------------------------------
// Step 4: Wait for SSH on remaining masters, join cluster via the VIP
// ---------------------------------------------------------------------------
const waitMaster2Ssh = new command.local.Command(
@@ -166,24 +196,24 @@ const joinMaster2 = new command.remote.Command(
"join-k3s-master-2",
{
connection: conn(master2Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-2`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-2`,
triggers: [master2VmId],
},
{ dependsOn: [waitMaster2Ssh] },
{ dependsOn: [waitMaster2Ssh, waitControlPlaneVipReady] },
);
const joinMaster3 = new command.remote.Command(
"join-k3s-master-3",
{
connection: conn(master3Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-3`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-3`,
triggers: [master3VmId],
},
{ dependsOn: [waitMaster3Ssh, joinMaster2] },
{ dependsOn: [waitMaster3Ssh, waitControlPlaneVipReady, joinMaster2] },
);
// ---------------------------------------------------------------------------
// Step 4: Join worker nodes
// Step 5: Join worker nodes via the VIP
// ---------------------------------------------------------------------------
const waitWorker1Ssh = new command.local.Command(
@@ -220,7 +250,7 @@ const joinWorker1 = new command.remote.Command(
"join-k3s-worker-1",
{
connection: conn(worker1Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
triggers: [worker1VmId],
},
{ dependsOn: [waitWorker1Ssh] },
@@ -230,7 +260,7 @@ const joinWorker2 = new command.remote.Command(
"join-k3s-worker-2",
{
connection: conn(worker2Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
triggers: [worker2VmId],
},
{ dependsOn: [waitWorker2Ssh] },
@@ -240,14 +270,14 @@ const joinWorker3 = new command.remote.Command(
"join-k3s-worker-3",
{
connection: conn(worker3Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`,
triggers: [worker3VmId],
},
{ dependsOn: [waitWorker3Ssh] },
);
// ---------------------------------------------------------------------------
// Step 5: Format and mount Longhorn disks
// Step 6: Format and mount Longhorn disks
// ---------------------------------------------------------------------------
const workers = [worker1Ip, worker2Ip, worker3Ip];
@@ -258,13 +288,13 @@ workers.forEach((worker, i) => {
`mount-longhorn-disk-${i}`,
{
connection: conn(worker),
create: `set -euo pipefail;
DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1');
DISK_PATH="/dev/disk/by-id/$DISK_ID";
sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH";
sudo mkdir -p /mnt/longhorn-extra;
echo ""$DISK_PATH" /mnt/longhorn-extra ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab;
sudo mount -a;
create: `set -euo pipefail;
DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1');
DISK_PATH="/dev/disk/by-id/$DISK_ID";
sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH";
sudo mkdir -p /mnt/longhorn-extra;
echo ""$DISK_PATH" /mnt/longhorn-extra ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab;
sudo mount -a;
df -h /mnt/longhorn-extra`,
},
{ dependsOn: workerSshReader[i] },
@@ -272,7 +302,7 @@ workers.forEach((worker, i) => {
});
// ---------------------------------------------------------------------------
// Step 6: Read kubeconfig from master-1, patch server URL for external access
// Step 7: Read kubeconfig from master-1, patch server URL to the VIP
// ---------------------------------------------------------------------------
const getKubeconfig = new command.remote.Command(
@@ -286,7 +316,7 @@ const getKubeconfig = new command.remote.Command(
);
// ---------------------------------------------------------------------------
// Step 7: Install additional packages
// Step 8: Install additional packages
// ---------------------------------------------------------------------------
const allVms = [
@@ -342,6 +372,6 @@ allVms.forEach(({ id, ip, name, readiness }) => {
export const kubeconfig = pulumi.secret(
pulumi
.all([getKubeconfig.stdout, master1Ip])
.apply(([kc, ip]) => kc.replaceAll("127.0.0.1", ip as string).trim()),
.all([getKubeconfig.stdout, controlPlaneVipAddress])
.apply(([kc, vip]) => kc.replaceAll("127.0.0.1", vip as string).trim()),
);