kasun/homelab-infrastructure-as-code
1
0 Fork 0
Code Issues 11 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

Feature/control plane vip via haproxy #34

Merged
kasun merged 2 commits from feature/control-plane-vip-via-haproxy into main 2026-10-03 00:02:12 +02:00
Conversation 0 Commits 2 Files Changed 2
+52 -22
1 changed files with 52 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit afac1f338f - Show all commits
02-k8s-bootstrap/index.ts
+52 -22
View File
@@ -37,6 +37,13 @@ const worker3Ip = infraRef.requireOutput("worker3Ip");
// Pre-shared k3s cluster token
const k3sToken = config.requireSecret("k3sToken");
// Control-plane VIP — floating IP for API server access, ARP-announced by
// kube-vip on whichever master is alive. Added as a TLS SAN on master-1's
// cert below, then used (instead of master1Ip) for every other node's join
// URL and for the exported kubeconfig, so the cluster stays reachable
// through the loss of any single master.
const controlPlaneVipAddress = config.requireSecret("controlPlaneVipAddress");
// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<
Record<string, number>
@@ -122,7 +129,7 @@ const installMaster1 = new command.remote.Command(
"install-k3s-master-1",
{
connection: conn(master1Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --disable servicelb --node-name k3s-master-1`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --tls-san ${controlPlaneVipAddress} --disable servicelb --node-name k3s-master-1`,
triggers: [master1VmId],
},
{ dependsOn: [waitMaster1Ssh] },
@@ -139,7 +146,30 @@ const waitK3sMaster1Ready = new command.remote.Command(
);
// ---------------------------------------------------------------------------
// Step 3: Wait for SSH on remaining masters, join cluster
// Step 3: Confirm the control-plane VIP is actually reachable before anyone
// else joins, so master-2/master-3 and every worker join through the VIP
// instead of master1Ip directly. The VIP itself is NOT managed here (or by
// any Pulumi stack) — it's a pfSense IP Alias Virtual IP with HAProxy
// TCP-load-balancing 6443 across all three masters, set up manually (see
// "pfSense one-time setup" in this stack's CLAUDE.md). This intentionally
// avoids a second kube-vip/ARP-based VIP for something as critical as API
// access — 03-k8s-infra's existing kube-vip DaemonSet already handles
// Service/LoadBalancer VIPs (e.g. Traefik) separately, entirely inside the
// cluster.
// ---------------------------------------------------------------------------
const waitControlPlaneVipReady = new command.remote.Command(
"wait-control-plane-vip-ready",
{
connection: conn(master1Ip),
create: pulumi.interpolate`for i in $(seq 1 60); do curl -sk --max-time 3 https://${controlPlaneVipAddress}:6443/healthz && exit 0; sleep 5; done; exit 1`,
triggers: [master1VmId],
},
{ dependsOn: [waitK3sMaster1Ready] },
);
// ---------------------------------------------------------------------------
// Step 4: Wait for SSH on remaining masters, join cluster via the VIP
// ---------------------------------------------------------------------------
const waitMaster2Ssh = new command.local.Command(
@@ -166,24 +196,24 @@ const joinMaster2 = new command.remote.Command(
"join-k3s-master-2",
{
connection: conn(master2Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-2`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-2`,
triggers: [master2VmId],
},
{ dependsOn: [waitMaster2Ssh] },
{ dependsOn: [waitMaster2Ssh, waitControlPlaneVipReady] },
);
const joinMaster3 = new command.remote.Command(
"join-k3s-master-3",
{
connection: conn(master3Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-3`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${controlPlaneVipAddress}:6443 --disable servicelb --node-name k3s-master-3`,
triggers: [master3VmId],
},
{ dependsOn: [waitMaster3Ssh, joinMaster2] },
{ dependsOn: [waitMaster3Ssh, waitControlPlaneVipReady, joinMaster2] },
);
// ---------------------------------------------------------------------------
// Step 4: Join worker nodes
// Step 5: Join worker nodes via the VIP
// ---------------------------------------------------------------------------
const waitWorker1Ssh = new command.local.Command(
@@ -220,7 +250,7 @@ const joinWorker1 = new command.remote.Command(
"join-k3s-worker-1",
{
connection: conn(worker1Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-1`,
triggers: [worker1VmId],
},
{ dependsOn: [waitWorker1Ssh] },
@@ -230,7 +260,7 @@ const joinWorker2 = new command.remote.Command(
"join-k3s-worker-2",
{
connection: conn(worker2Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-2`,
triggers: [worker2VmId],
},
{ dependsOn: [waitWorker2Ssh] },
@@ -240,14 +270,14 @@ const joinWorker3 = new command.remote.Command(
"join-k3s-worker-3",
{
connection: conn(worker3Ip),
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`,
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${controlPlaneVipAddress}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`,
triggers: [worker3VmId],
},
{ dependsOn: [waitWorker3Ssh] },
);
// ---------------------------------------------------------------------------
// Step 5: Format and mount Longhorn disks
// Step 6: Format and mount Longhorn disks
// ---------------------------------------------------------------------------
const workers = [worker1Ip, worker2Ip, worker3Ip];
@@ -258,13 +288,13 @@ workers.forEach((worker, i) => {
`mount-longhorn-disk-${i}`,
{
connection: conn(worker),
create: `set -euo pipefail;
DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1');
DISK_PATH="/dev/disk/by-id/$DISK_ID";
sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH";
sudo mkdir -p /mnt/longhorn-extra;
echo ""$DISK_PATH" /mnt/longhorn-extra ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab;
sudo mount -a;
create: `set -euo pipefail;
DISK_ID=$(ls /dev/disk/by-id/ | grep 'scsi1');
DISK_PATH="/dev/disk/by-id/$DISK_ID";
sudo blkid "$DISK_PATH" >/dev/null 2>&1 || sudo mkfs.ext4 "$DISK_PATH";
sudo mkdir -p /mnt/longhorn-extra;
echo ""$DISK_PATH" /mnt/longhorn-extra ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab;
sudo mount -a;
df -h /mnt/longhorn-extra`,
},
{ dependsOn: workerSshReader[i] },
@@ -272,7 +302,7 @@ workers.forEach((worker, i) => {
});
// ---------------------------------------------------------------------------
// Step 6: Read kubeconfig from master-1, patch server URL for external access
// Step 7: Read kubeconfig from master-1, patch server URL to the VIP
// ---------------------------------------------------------------------------
const getKubeconfig = new command.remote.Command(
@@ -286,7 +316,7 @@ const getKubeconfig = new command.remote.Command(
);
// ---------------------------------------------------------------------------
// Step 7: Install additional packages
// Step 8: Install additional packages
// ---------------------------------------------------------------------------
const allVms = [
@@ -342,6 +372,6 @@ allVms.forEach(({ id, ip, name, readiness }) => {
export const kubeconfig = pulumi.secret(
pulumi
.all([getKubeconfig.stdout, master1Ip])
.apply(([kc, ip]) => kc.replaceAll("127.0.0.1", ip as string).trim()),
.all([getKubeconfig.stdout, controlPlaneVipAddress])
.apply(([kc, vip]) => kc.replaceAll("127.0.0.1", vip as string).trim()),
);