added k3s bootstrap config to deploy to all dedicated nodes.
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
name: Deploy k8s Bootstrap
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'k8s-bootstrap/**'
|
||||
- '.gitea/workflows/deploy-k8s-bootstrap.yaml'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'k8s-bootstrap/**'
|
||||
- '.gitea/workflows/deploy-k8s-bootstrap.yaml'
|
||||
|
||||
jobs:
|
||||
preview:
|
||||
name: Pulumi Preview
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '24'
|
||||
|
||||
- name: Restore Stack Config
|
||||
run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
|
||||
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
working-directory: k8s-bootstrap
|
||||
|
||||
- name: Preview
|
||||
uses: pulumi/actions@v5
|
||||
with:
|
||||
command: preview
|
||||
stack-name: dev
|
||||
work-dir: k8s-bootstrap
|
||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||
env:
|
||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||
|
||||
deploy:
|
||||
name: Bootstrap k3s Cluster
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '24'
|
||||
|
||||
- name: Restore Stack Config
|
||||
run: echo "${{ secrets.K8S_BOOTSTRAP_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
|
||||
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
working-directory: k8s-bootstrap
|
||||
|
||||
- name: Refresh State
|
||||
uses: pulumi/actions@v5
|
||||
with:
|
||||
command: refresh
|
||||
stack-name: dev
|
||||
work-dir: k8s-bootstrap
|
||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||
env:
|
||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||
|
||||
- name: Deploy
|
||||
uses: pulumi/actions@v5
|
||||
with:
|
||||
command: up
|
||||
stack-name: dev
|
||||
work-dir: k8s-bootstrap
|
||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||
env:
|
||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||
|
||||
# Propagate kubeconfig to the downstream stacks so their next deploy picks it up
|
||||
- name: Propagate kubeconfig to k8s-infra
|
||||
run: |
|
||||
echo "${{ secrets.K8S_INFRA_DEV_YAML }}" | base64 -d > k8s-infra/Pulumi.dev.yaml
|
||||
cd k8s-infra && npm install
|
||||
KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \
|
||||
--cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev)
|
||||
pulumi config set --secret kubeconfig "$KUBECONFIG" \
|
||||
--cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev
|
||||
# Re-encode updated config for the secret (update manually in Gitea after first run)
|
||||
base64 -w 0 Pulumi.dev.yaml
|
||||
working-directory: .
|
||||
env:
|
||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||
|
||||
- name: Propagate kubeconfig to k8s-apps
|
||||
run: |
|
||||
echo "${{ secrets.K8S_APPS_DEV_YAML }}" | base64 -d > k8s-apps/Pulumi.dev.yaml
|
||||
cd k8s-apps && npm install
|
||||
KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets \
|
||||
--cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev)
|
||||
pulumi config set --secret kubeconfig "$KUBECONFIG" \
|
||||
--cloud-url "${{ secrets.PULUMI_BACKEND_URL }}" -s dev
|
||||
base64 -w 0 Pulumi.dev.yaml
|
||||
working-directory: .
|
||||
env:
|
||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||
@@ -0,0 +1,10 @@
|
||||
name: k8s-bootstrap
|
||||
description: Bootstrap k3s cluster on Proxmox VMs via QEMU guest exec
|
||||
runtime:
|
||||
name: nodejs
|
||||
options:
|
||||
packagemanager: npm
|
||||
config:
|
||||
pulumi:tags:
|
||||
value:
|
||||
pulumi:template: typescript
|
||||
@@ -0,0 +1,143 @@
|
||||
import * as pulumi from "@pulumi/pulumi";
|
||||
import * as command from "@pulumi/command";
|
||||
|
||||
const config = new pulumi.Config();
|
||||
|
||||
// Proxmox API credentials — same as proxmox-infra stack
|
||||
const pve1Endpoint = config.requireSecret("pve1Endpoint");
|
||||
const pve1ApiToken = config.requireSecret("pve1ApiToken");
|
||||
const pve2Endpoint = config.requireSecret("pve2Endpoint");
|
||||
const pve2ApiToken = config.requireSecret("pve2ApiToken");
|
||||
|
||||
// Node IPs — static DHCP leases set in the router
|
||||
const master1Ip = config.require("master1Ip");
|
||||
const master2Ip = config.require("master2Ip");
|
||||
const master3Ip = config.require("master3Ip");
|
||||
const worker1Ip = config.require("worker1Ip");
|
||||
const worker2Ip = config.require("worker2Ip");
|
||||
|
||||
// Pre-shared k3s cluster token
|
||||
const k3sToken = config.requireSecret("k3sToken");
|
||||
|
||||
// VM IDs and CI runner SSH key — read from proxmox-infra stack outputs
|
||||
const infraRef = new pulumi.StackReference(`${pulumi.getOrganization()}/proxmox-infra/dev`);
|
||||
const vmIdsOutput = infraRef.requireOutput("vmIds") as pulumi.Output<Record<string, number>>;
|
||||
const ciRunnerPrivateKey = infraRef.requireOutput("ciRunnerPrivateKey") as pulumi.Output<string>;
|
||||
|
||||
const master1VmId = vmIdsOutput.apply(ids => String(ids.master1));
|
||||
const master2VmId = vmIdsOutput.apply(ids => String(ids.master2));
|
||||
const master3VmId = vmIdsOutput.apply(ids => String(ids.master3));
|
||||
const worker1VmId = vmIdsOutput.apply(ids => String(ids.worker1));
|
||||
const worker2VmId = vmIdsOutput.apply(ids => String(ids.worker2));
|
||||
|
||||
// SSH connection helper
|
||||
function conn(ip: string): command.types.input.remote.ConnectionArgs {
|
||||
return { host: ip, user: "ubuntu", privateKey: ciRunnerPrivateKey };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step 1: Start all VMs (fire-and-forget — VMs are owned by proxmox-infra)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const startMaster1 = new command.local.Command("start-master-1", {
|
||||
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master1VmId}/status/start" 2>/dev/null || true`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
});
|
||||
const startMaster2 = new command.local.Command("start-master-2", {
|
||||
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${master2VmId}/status/start" 2>/dev/null || true`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
});
|
||||
const startWorker1 = new command.local.Command("start-worker-1", {
|
||||
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve1ApiToken}" "${pve1Endpoint}/api2/json/nodes/pve/qemu/${worker1VmId}/status/start" 2>/dev/null || true`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
});
|
||||
const startMaster3 = new command.local.Command("start-master-3", {
|
||||
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${master3VmId}/status/start" 2>/dev/null || true`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
});
|
||||
const startWorker2 = new command.local.Command("start-worker-2", {
|
||||
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve2ApiToken}" "${pve2Endpoint}/api2/json/nodes/pve-bckp/qemu/${worker2VmId}/status/start" 2>/dev/null || true`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
});
|
||||
|
||||
const allStarts = [startMaster1, startMaster2, startMaster3, startWorker1, startWorker2];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step 2: Wait for SSH on master-1, install k3s
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const waitMaster1Ssh = new command.local.Command("wait-ssh-master-1", {
|
||||
create: `for i in $(seq 1 60); do nc -z -w 5 ${master1Ip} 22 && exit 0; sleep 5; done; exit 1`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
}, { dependsOn: allStarts });
|
||||
|
||||
const installMaster1 = new command.remote.Command("install-k3s-master-1", {
|
||||
connection: conn(master1Ip),
|
||||
create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip}`,
|
||||
}, { dependsOn: [waitMaster1Ssh] });
|
||||
|
||||
const waitK3sMaster1Ready = new command.remote.Command("wait-k3s-master-1-ready", {
|
||||
connection: conn(master1Ip),
|
||||
create: `for i in $(seq 1 60); do kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get node k3s-master-1 --no-headers 2>/dev/null | grep -q " Ready" && exit 0; sleep 10; done; exit 1`,
|
||||
}, { dependsOn: [installMaster1] });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step 3: Wait for SSH on remaining masters, join cluster
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const waitMaster2Ssh = new command.local.Command("wait-ssh-master-2", {
|
||||
create: `for i in $(seq 1 60); do nc -z -w 5 ${master2Ip} 22 && exit 0; sleep 5; done; exit 1`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
}, { dependsOn: [waitK3sMaster1Ready] });
|
||||
|
||||
const waitMaster3Ssh = new command.local.Command("wait-ssh-master-3", {
|
||||
create: `for i in $(seq 1 60); do nc -z -w 5 ${master3Ip} 22 && exit 0; sleep 5; done; exit 1`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
}, { dependsOn: [waitK3sMaster1Ready] });
|
||||
|
||||
const joinMaster2 = new command.remote.Command("join-k3s-master-2", {
|
||||
connection: conn(master2Ip),
|
||||
create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
|
||||
}, { dependsOn: [waitMaster2Ssh] });
|
||||
|
||||
const joinMaster3 = new command.remote.Command("join-k3s-master-3", {
|
||||
connection: conn(master3Ip),
|
||||
create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443`,
|
||||
}, { dependsOn: [waitMaster3Ssh, joinMaster2] });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step 4: Join worker nodes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const waitWorker1Ssh = new command.local.Command("wait-ssh-worker-1", {
|
||||
create: `for i in $(seq 1 60); do nc -z -w 5 ${worker1Ip} 22 && exit 0; sleep 5; done; exit 1`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
}, { dependsOn: [joinMaster3] });
|
||||
|
||||
const waitWorker2Ssh = new command.local.Command("wait-ssh-worker-2", {
|
||||
create: `for i in $(seq 1 60); do nc -z -w 5 ${worker2Ip} 22 && exit 0; sleep 5; done; exit 1`,
|
||||
interpreter: ["/bin/bash", "-c"],
|
||||
}, { dependsOn: [joinMaster3] });
|
||||
|
||||
const joinWorker1 = new command.remote.Command("join-k3s-worker-1", {
|
||||
connection: conn(worker1Ip),
|
||||
create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
|
||||
}, { dependsOn: [waitWorker1Ssh] });
|
||||
|
||||
const joinWorker2 = new command.remote.Command("join-k3s-worker-2", {
|
||||
connection: conn(worker2Ip),
|
||||
create: pulumi.interpolate`curl -sfL https://get.k3s.io | K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s -`,
|
||||
}, { dependsOn: [waitWorker2Ssh] });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step 5: Read kubeconfig from master-1, patch server URL for external access
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const getKubeconfig = new command.remote.Command("get-kubeconfig", {
|
||||
connection: conn(master1Ip),
|
||||
create: `cat /etc/rancher/k3s/k3s.yaml`,
|
||||
}, { dependsOn: [joinWorker1, joinWorker2] });
|
||||
|
||||
export const kubeconfig = pulumi.secret(
|
||||
getKubeconfig.stdout.apply(kc => kc.replace(/127\.0\.0\.1/g, master1Ip).trim()),
|
||||
);
|
||||
Generated
+2670
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "k8s-bootstrap",
|
||||
"main": "index.ts",
|
||||
"devDependencies": {
|
||||
"@types/node": "^18",
|
||||
"typescript": "^5.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pulumi/command": "^0.11.0",
|
||||
"@pulumi/pulumi": "^3.113.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"strict": true,
|
||||
"outDir": "bin",
|
||||
"target": "es2020",
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"sourceMap": true,
|
||||
"experimentalDecorators": true,
|
||||
"pretty": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noImplicitReturns": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"files": [
|
||||
"index.ts"
|
||||
]
|
||||
}
|
||||
+18
-2
@@ -1,5 +1,6 @@
|
||||
import * as pulumi from "@pulumi/pulumi";
|
||||
import * as proxmox from "@muhlba91/pulumi-proxmoxve";
|
||||
import * as tls from "@pulumi/tls";
|
||||
|
||||
const config = new pulumi.Config();
|
||||
|
||||
@@ -19,6 +20,15 @@ const pveBckpProvider = new proxmox.Provider("pve-bckp", {
|
||||
insecure: true,
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CI runner SSH keypair — generated once, stored in Pulumi state backend.
|
||||
// Public key goes into every VM; private key is exported for k8s-bootstrap.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ciRunnerKey = new tls.PrivateKey("ci-runner-key", {
|
||||
algorithm: "ED25519",
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Download Ubuntu Noble cloud image to each node's ISO storage
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -219,7 +229,10 @@ const k3sVms = nodeConfigs.map(
|
||||
userAccount: {
|
||||
username: "ubuntu",
|
||||
password: k3sVmPassword,
|
||||
keys: [sshPvePublicKey.apply((k) => k.trim())],
|
||||
keys: [
|
||||
sshPvePublicKey.apply((k) => k.trim()),
|
||||
ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
|
||||
],
|
||||
},
|
||||
},
|
||||
networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
|
||||
@@ -245,7 +258,7 @@ export const clusterInfo = k3sVms.map((vm, index) => ({
|
||||
role: nodeConfigs[index].role,
|
||||
}));
|
||||
|
||||
// Individual vmId exports — used by k8s-bootstrap to start VMs and run guest exec.
|
||||
// Individual vmId exports — used by k8s-bootstrap to start VMs.
|
||||
// Order matches nodeConfigs: master-1, master-2, worker-1, master-3, worker-2.
|
||||
export const vmIds = {
|
||||
master1: k3sVms[0].vmId,
|
||||
@@ -254,3 +267,6 @@ export const vmIds = {
|
||||
master3: k3sVms[3].vmId,
|
||||
worker2: k3sVms[4].vmId,
|
||||
};
|
||||
|
||||
// CI runner SSH private key — consumed by k8s-bootstrap via StackReference.
|
||||
export const ciRunnerPrivateKey = pulumi.secret(ciRunnerKey.privateKeyOpenssh);
|
||||
|
||||
Generated
+11
-1
@@ -7,7 +7,8 @@
|
||||
"name": "proxmox-infra",
|
||||
"dependencies": {
|
||||
"@muhlba91/pulumi-proxmoxve": "^8.2.1",
|
||||
"@pulumi/pulumi": "^3.113.0"
|
||||
"@pulumi/pulumi": "^3.113.0",
|
||||
"@pulumi/tls": "^5.5.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^18",
|
||||
@@ -752,6 +753,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@pulumi/tls": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@pulumi/tls/-/tls-5.5.0.tgz",
|
||||
"integrity": "sha512-OTGxp4sgDEuXlXrd7NtxrhhciPgcn9rqDIZlGOTmTaeGo+tlVMwv73FFqiVrzAmBiILdU8tLXHfxhjc+bTLdrQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@pulumi/pulumi": "^3.142.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@sigstore/bundle": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sigstore/bundle/-/bundle-4.0.0.tgz",
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@muhlba91/pulumi-proxmoxve": "^8.2.1",
|
||||
"@pulumi/pulumi": "^3.113.0"
|
||||
"@pulumi/pulumi": "^3.113.0",
|
||||
"@pulumi/tls": "^5.5.0"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user