Compare commits
8
Commits
ca940be8fa
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f57f1822f | ||
|
|
07c4678a6d | ||
|
|
fb1f8db36b | ||
|
|
0ae688ac66 | ||
|
|
b7d5a5c5cd | ||
|
|
10a4578ce4 | ||
|
|
3f98874008 | ||
|
|
e830eb3b40 |
+133
-195
@@ -25,33 +25,121 @@ const worker1Ip = config.requireSecret("worker1Ip");
|
|||||||
const worker2Ip = config.requireSecret("worker2Ip");
|
const worker2Ip = config.requireSecret("worker2Ip");
|
||||||
const worker3Ip = config.requireSecret("worker3Ip");
|
const worker3Ip = config.requireSecret("worker3Ip");
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// VM templates — one per node, cloned from the downloaded cloud image.
|
||||||
|
// Templates are not started and have no cloud-init config; that is applied
|
||||||
|
// per-clone so each node gets its own hostname (derived from VM name).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const templateSettings = {
|
||||||
|
template: true,
|
||||||
|
started: false,
|
||||||
|
stopOnDestroy: true,
|
||||||
|
scsiHardware: "virtio-scsi-pci",
|
||||||
|
cdrom: { fileId: "none" },
|
||||||
|
cpu: {
|
||||||
|
cores: 2,
|
||||||
|
sockets: 1,
|
||||||
|
type: "host",
|
||||||
|
numa: true,
|
||||||
|
},
|
||||||
|
memory: {
|
||||||
|
dedicated: 2048,
|
||||||
|
floating: 0,
|
||||||
|
},
|
||||||
|
networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
|
||||||
|
serialDevices: [{}],
|
||||||
|
vga: { type: "serial0" },
|
||||||
|
agent: { enabled: true, timeout: "30s" },
|
||||||
|
};
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Providers — one per standalone Proxmox machine
|
// Providers — one per standalone Proxmox machine
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const pveProvider = new proxmox.Provider("pve", {
|
interface PveNodeConfig {
|
||||||
endpoint: pve1Endpoint,
|
nodeName: string;
|
||||||
apiToken: pve1ApiToken,
|
hostLabel: string;
|
||||||
insecure: true,
|
isoStore: string;
|
||||||
});
|
dataStore: string;
|
||||||
|
endpoint: pulumi.Output<string>;
|
||||||
|
apiToken: pulumi.Output<string>;
|
||||||
|
}
|
||||||
|
|
||||||
const pveBckpProvider = new proxmox.Provider("pve-bckp", {
|
const pveNodeSeeds: PveNodeConfig[] = [
|
||||||
endpoint: pve2Endpoint,
|
{
|
||||||
apiToken: pve2ApiToken,
|
nodeName: "pve",
|
||||||
insecure: true,
|
hostLabel: "pve-b550",
|
||||||
});
|
isoStore: "pve-local-ext1",
|
||||||
|
dataStore: "local-lvm",
|
||||||
|
endpoint: pve1Endpoint,
|
||||||
|
apiToken: pve1ApiToken,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeName: "pve-bckp",
|
||||||
|
hostLabel: "pve-optiplex",
|
||||||
|
isoStore: "local",
|
||||||
|
dataStore: "local",
|
||||||
|
endpoint: pve2Endpoint,
|
||||||
|
apiToken: pve2ApiToken,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeName: "pve",
|
||||||
|
hostLabel: "pve-esprimo",
|
||||||
|
isoStore: "local",
|
||||||
|
dataStore: "local-lvm",
|
||||||
|
endpoint: pve3Endpoint,
|
||||||
|
apiToken: pve3ApiToken,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
const pveEsprimoProvider = new proxmox.Provider("pve-esprimo", {
|
const pveNodes = pveNodeSeeds
|
||||||
endpoint: pve3Endpoint,
|
.map((n) => ({
|
||||||
apiToken: pve3ApiToken,
|
...n,
|
||||||
insecure: true,
|
provider: new proxmox.Provider(n.hostLabel, {
|
||||||
});
|
endpoint: n.endpoint,
|
||||||
|
apiToken: n.apiToken,
|
||||||
const pveNodes = {
|
insecure: true,
|
||||||
"pve-b550": { nodeName: "pve", isoStore: "pve-local-ext1", dataStore: "local-lvm" },
|
}),
|
||||||
"pve-optiplex": { nodeName: "pve-bckp", isoStore: "local", dataStore: "local" },
|
}))
|
||||||
"pve-esprimo": { nodeName: "pve", isoStore: "local", dataStore: "local-lvm" },
|
.map((n) => ({
|
||||||
} as const;
|
...n,
|
||||||
|
ubuntuImagePve: new proxmox.download.File(
|
||||||
|
`ubuntu-noble-${n.hostLabel}`,
|
||||||
|
{
|
||||||
|
nodeName: n.nodeName,
|
||||||
|
datastoreId: n.isoStore,
|
||||||
|
contentType: "import",
|
||||||
|
fileName: "noble-server-cloudimg-amd64.qcow2",
|
||||||
|
url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img",
|
||||||
|
overwrite: false,
|
||||||
|
overwriteUnmanaged: true,
|
||||||
|
},
|
||||||
|
{ provider: n.provider },
|
||||||
|
),
|
||||||
|
}))
|
||||||
|
.map((n) => ({
|
||||||
|
...n,
|
||||||
|
pveTemplate: new proxmox.VmLegacy(
|
||||||
|
`k3s-template-${n.hostLabel}`,
|
||||||
|
{
|
||||||
|
...templateSettings,
|
||||||
|
nodeName: n.nodeName,
|
||||||
|
name: "k3s-ubuntu-noble-template",
|
||||||
|
disks: [
|
||||||
|
{
|
||||||
|
interface: "scsi0",
|
||||||
|
datastoreId: n.dataStore,
|
||||||
|
importFrom: pulumi.interpolate`${n.ubuntuImagePve.datastoreId}:import/${n.ubuntuImagePve.fileName}`,
|
||||||
|
size: 10,
|
||||||
|
ssd: true,
|
||||||
|
discard: "on",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{ provider: n.provider, ignoreChanges: ["disks"] },
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Providers — PfSense
|
// Providers — PfSense
|
||||||
@@ -73,215 +161,62 @@ const ciRunnerKey = new tls.PrivateKey("ci-runner-key", {
|
|||||||
algorithm: "ED25519",
|
algorithm: "ED25519",
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Download Ubuntu Noble cloud image to each node's ISO storage
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
const ubuntuNobleUrl =
|
|
||||||
"https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img";
|
|
||||||
|
|
||||||
const ubuntuImagePve = new proxmox.download.File(
|
|
||||||
"ubuntu-noble-pve",
|
|
||||||
{
|
|
||||||
nodeName: pveNodes["pve-b550"].nodeName,
|
|
||||||
datastoreId: pveNodes["pve-b550"].isoStore,
|
|
||||||
contentType: "import",
|
|
||||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
|
||||||
url: ubuntuNobleUrl,
|
|
||||||
overwrite: false,
|
|
||||||
overwriteUnmanaged: true,
|
|
||||||
},
|
|
||||||
{ provider: pveProvider },
|
|
||||||
);
|
|
||||||
|
|
||||||
const ubuntuImagePveBckp = new proxmox.download.File(
|
|
||||||
"ubuntu-noble-pve-bckp",
|
|
||||||
{
|
|
||||||
nodeName: pveNodes["pve-optiplex"].nodeName,
|
|
||||||
datastoreId: pveNodes["pve-optiplex"].isoStore,
|
|
||||||
contentType: "import",
|
|
||||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
|
||||||
url: ubuntuNobleUrl,
|
|
||||||
overwrite: false,
|
|
||||||
overwriteUnmanaged: true,
|
|
||||||
},
|
|
||||||
{ provider: pveBckpProvider },
|
|
||||||
);
|
|
||||||
|
|
||||||
const ubuntuImagePveEsprimo = new proxmox.download.File(
|
|
||||||
"ubuntu-noble-pve-esprimo",
|
|
||||||
{
|
|
||||||
nodeName: pveNodes["pve-esprimo"].nodeName,
|
|
||||||
datastoreId: pveNodes["pve-esprimo"].isoStore,
|
|
||||||
contentType: "import",
|
|
||||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
|
||||||
url: ubuntuNobleUrl,
|
|
||||||
overwrite: false,
|
|
||||||
overwriteUnmanaged: true,
|
|
||||||
},
|
|
||||||
{ provider: pveEsprimoProvider },
|
|
||||||
);
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// VM templates — one per node, cloned from the downloaded cloud image.
|
|
||||||
// Templates are not started and have no cloud-init config; that is applied
|
|
||||||
// per-clone so each node gets its own hostname (derived from VM name).
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
const templateSettings = {
|
|
||||||
template: true,
|
|
||||||
started: false,
|
|
||||||
stopOnDestroy: true,
|
|
||||||
scsiHardware: "virtio-scsi-pci",
|
|
||||||
cpu: {
|
|
||||||
cores: 2,
|
|
||||||
sockets: 1,
|
|
||||||
type: "host",
|
|
||||||
numa: true,
|
|
||||||
},
|
|
||||||
memory: {
|
|
||||||
dedicated: 2048,
|
|
||||||
floating: 0,
|
|
||||||
},
|
|
||||||
networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
|
|
||||||
serialDevices: [{}],
|
|
||||||
vga: { type: "serial0" },
|
|
||||||
agent: { enabled: true },
|
|
||||||
};
|
|
||||||
|
|
||||||
const pveTemplate = new proxmox.VmLegacy(
|
|
||||||
"k3s-template-pve",
|
|
||||||
{
|
|
||||||
...templateSettings,
|
|
||||||
nodeName: "pve",
|
|
||||||
name: "k3s-ubuntu-noble-template",
|
|
||||||
disks: [
|
|
||||||
{
|
|
||||||
interface: "scsi0",
|
|
||||||
datastoreId: pveNodes["pve-b550"].dataStore,
|
|
||||||
importFrom: pulumi.interpolate`${ubuntuImagePve.datastoreId}:import/${ubuntuImagePve.fileName}`,
|
|
||||||
size: 10,
|
|
||||||
ssd: true,
|
|
||||||
discard: "on",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{ provider: pveProvider, ignoreChanges: ["disks"] },
|
|
||||||
);
|
|
||||||
|
|
||||||
const pveBckpTemplate = new proxmox.VmLegacy(
|
|
||||||
"k3s-template-pve-bckp",
|
|
||||||
{
|
|
||||||
...templateSettings,
|
|
||||||
nodeName: "pve-bckp",
|
|
||||||
name: "k3s-ubuntu-noble-template",
|
|
||||||
disks: [
|
|
||||||
{
|
|
||||||
interface: "scsi0",
|
|
||||||
datastoreId: pveNodes["pve-optiplex"].dataStore,
|
|
||||||
importFrom: pulumi.interpolate`${ubuntuImagePveBckp.datastoreId}:import/${ubuntuImagePveBckp.fileName}`,
|
|
||||||
size: 10,
|
|
||||||
ssd: true,
|
|
||||||
discard: "on",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{ provider: pveBckpProvider, ignoreChanges: ["disks"] },
|
|
||||||
);
|
|
||||||
|
|
||||||
const pveEsprimoTemplate = new proxmox.VmLegacy(
|
|
||||||
"k3s-template-pve-esprimo",
|
|
||||||
{
|
|
||||||
...templateSettings,
|
|
||||||
nodeName: "pve",
|
|
||||||
name: "k3s-ubuntu-noble-template",
|
|
||||||
disks: [
|
|
||||||
{
|
|
||||||
interface: "scsi0",
|
|
||||||
datastoreId: pveNodes["pve-esprimo"].dataStore,
|
|
||||||
importFrom: pulumi.interpolate`${ubuntuImagePveEsprimo.datastoreId}:import/${ubuntuImagePveEsprimo.fileName}`,
|
|
||||||
size: 10,
|
|
||||||
ssd: true,
|
|
||||||
discard: "on",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{ provider: pveEsprimoProvider, ignoreChanges: ["disks"] },
|
|
||||||
);
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// k3s nodes — full clones of their respective templates
|
// k3s nodes — full clones of their respective templates
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const k3sVmPassword = config.requireSecret("k3sVmPassword");
|
const k3sVmPassword = config.requireSecret("k3sVmPassword");
|
||||||
const sshPvePublicKey = config.requireSecret("sshPvePublicKey");
|
const pve1SshPublicKey = config.requireSecret("pve1SshPublicKey");
|
||||||
|
const pve2SshPublicKey = config.requireSecret("pve2SshPublicKey");
|
||||||
|
const pve3SshPublicKey = config.requireSecret("pve3SshPublicKey");
|
||||||
|
|
||||||
interface NodeConfig {
|
interface NodeConfig {
|
||||||
name: string;
|
name: string;
|
||||||
role: "master" | "worker";
|
role: "master" | "worker";
|
||||||
nodeName: string;
|
host: (typeof pveNodes)[number]; // one row from the pveNodes chain
|
||||||
provider: proxmox.Provider;
|
|
||||||
template: proxmox.VmLegacy;
|
|
||||||
diskDatastore: string;
|
|
||||||
ip: pulumi.Output<string>;
|
ip: pulumi.Output<string>;
|
||||||
longhornDiskSize?: number;
|
longhornDiskSize?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const hostFor = (label: string) => pveNodes.find((n) => n.hostLabel === label)!;
|
||||||
|
|
||||||
const nodeConfigs: NodeConfig[] = [
|
const nodeConfigs: NodeConfig[] = [
|
||||||
{
|
{
|
||||||
name: "k3s-master-1",
|
name: "k3s-master-1",
|
||||||
role: "master",
|
role: "master",
|
||||||
nodeName: pveNodes["pve-b550"].nodeName,
|
host: hostFor("pve-b550"),
|
||||||
provider: pveProvider,
|
|
||||||
template: pveTemplate,
|
|
||||||
diskDatastore: pveNodes["pve-b550"].dataStore,
|
|
||||||
ip: master1Ip,
|
ip: master1Ip,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "k3s-master-2",
|
name: "k3s-master-2",
|
||||||
role: "master",
|
role: "master",
|
||||||
nodeName: pveNodes["pve-esprimo"].nodeName,
|
host: hostFor("pve-esprimo"),
|
||||||
provider: pveEsprimoProvider,
|
|
||||||
template: pveTemplate,
|
|
||||||
diskDatastore: pveNodes["pve-esprimo"].dataStore,
|
|
||||||
ip: master2Ip,
|
ip: master2Ip,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "k3s-worker-1",
|
name: "k3s-worker-1",
|
||||||
role: "worker",
|
role: "worker",
|
||||||
nodeName: pveNodes["pve-b550"].nodeName,
|
host: hostFor("pve-b550"),
|
||||||
provider: pveProvider,
|
|
||||||
template: pveTemplate,
|
|
||||||
diskDatastore: pveNodes["pve-b550"].dataStore,
|
|
||||||
ip: worker1Ip,
|
ip: worker1Ip,
|
||||||
longhornDiskSize: 50,
|
longhornDiskSize: 50,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "k3s-master-3",
|
name: "k3s-master-3",
|
||||||
role: "master",
|
role: "master",
|
||||||
nodeName: pveNodes["pve-optiplex"].nodeName,
|
host: hostFor("pve-optiplex"),
|
||||||
provider: pveBckpProvider,
|
|
||||||
template: pveBckpTemplate,
|
|
||||||
diskDatastore: pveNodes["pve-optiplex"].dataStore,
|
|
||||||
ip: master3Ip,
|
ip: master3Ip,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "k3s-worker-2",
|
name: "k3s-worker-2",
|
||||||
role: "worker",
|
role: "worker",
|
||||||
nodeName: pveNodes["pve-optiplex"].nodeName,
|
host: hostFor("pve-optiplex"),
|
||||||
provider: pveBckpProvider,
|
|
||||||
template: pveBckpTemplate,
|
|
||||||
diskDatastore: pveNodes["pve-optiplex"].dataStore,
|
|
||||||
ip: worker2Ip,
|
ip: worker2Ip,
|
||||||
longhornDiskSize: 50,
|
longhornDiskSize: 50,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "k3s-worker-3",
|
name: "k3s-worker-3",
|
||||||
role: "worker",
|
role: "worker",
|
||||||
nodeName: pveNodes["pve-esprimo"].nodeName,
|
host: hostFor("pve-esprimo"),
|
||||||
provider: pveEsprimoProvider,
|
|
||||||
template: pveEsprimoTemplate,
|
|
||||||
diskDatastore: pveNodes["pve-esprimo"].dataStore,
|
|
||||||
ip: worker3Ip,
|
ip: worker3Ip,
|
||||||
longhornDiskSize: 50,
|
longhornDiskSize: 50,
|
||||||
},
|
},
|
||||||
@@ -292,14 +227,14 @@ const k3sVms = nodeConfigs.map(
|
|||||||
new proxmox.VmLegacy(
|
new proxmox.VmLegacy(
|
||||||
node.name,
|
node.name,
|
||||||
{
|
{
|
||||||
nodeName: node.nodeName,
|
nodeName: node.host.nodeName,
|
||||||
name: node.name,
|
name: node.name,
|
||||||
description: "k3s " + node.role + " node — managed by Pulumi",
|
description: "k3s " + node.role + " node — managed by Pulumi",
|
||||||
tags: ["k3s", node.role],
|
tags: ["k3s", node.role],
|
||||||
clone: {
|
clone: {
|
||||||
vmId: node.template.vmId,
|
vmId: node.host.pveTemplate.vmId,
|
||||||
full: true,
|
full: true,
|
||||||
datastoreId: node.diskDatastore,
|
datastoreId: node.host.dataStore,
|
||||||
},
|
},
|
||||||
cpu: {
|
cpu: {
|
||||||
cores: 2,
|
cores: 2,
|
||||||
@@ -314,7 +249,7 @@ const k3sVms = nodeConfigs.map(
|
|||||||
disks: [
|
disks: [
|
||||||
{
|
{
|
||||||
interface: "scsi0",
|
interface: "scsi0",
|
||||||
datastoreId: node.diskDatastore,
|
datastoreId: node.host.dataStore,
|
||||||
size: 35,
|
size: 35,
|
||||||
ssd: true,
|
ssd: true,
|
||||||
discard: "on",
|
discard: "on",
|
||||||
@@ -324,7 +259,7 @@ const k3sVms = nodeConfigs.map(
|
|||||||
: [
|
: [
|
||||||
{
|
{
|
||||||
interface: "scsi1",
|
interface: "scsi1",
|
||||||
datastoreId: node.diskDatastore,
|
datastoreId: node.host.dataStore,
|
||||||
size: node.longhornDiskSize,
|
size: node.longhornDiskSize,
|
||||||
ssd: true,
|
ssd: true,
|
||||||
discard: "on",
|
discard: "on",
|
||||||
@@ -332,13 +267,15 @@ const k3sVms = nodeConfigs.map(
|
|||||||
]),
|
]),
|
||||||
],
|
],
|
||||||
initialization: {
|
initialization: {
|
||||||
datastoreId: node.diskDatastore,
|
datastoreId: node.host.dataStore,
|
||||||
ipConfigs: [{ ipv4: { address: "dhcp" } }],
|
ipConfigs: [{ ipv4: { address: "dhcp" } }],
|
||||||
userAccount: {
|
userAccount: {
|
||||||
username: "ubuntu",
|
username: "ubuntu",
|
||||||
password: k3sVmPassword,
|
password: k3sVmPassword,
|
||||||
keys: [
|
keys: [
|
||||||
sshPvePublicKey.apply((k) => k.trim()),
|
pve1SshPublicKey.apply((k) => k.trim()),
|
||||||
|
pve2SshPublicKey.apply((k) => k.trim()),
|
||||||
|
pve3SshPublicKey.apply((k) => k.trim()),
|
||||||
ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
|
ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -347,12 +284,13 @@ const k3sVms = nodeConfigs.map(
|
|||||||
scsiHardware: "virtio-scsi-pci",
|
scsiHardware: "virtio-scsi-pci",
|
||||||
serialDevices: [{}],
|
serialDevices: [{}],
|
||||||
vga: { type: "serial0" },
|
vga: { type: "serial0" },
|
||||||
agent: { enabled: true },
|
agent: { enabled: true, timeout: "30s" },
|
||||||
|
cdrom: { fileId: "none" },
|
||||||
started: false,
|
started: false,
|
||||||
stopOnDestroy: true,
|
stopOnDestroy: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
provider: node.provider,
|
provider: node.host.provider,
|
||||||
retainOnDelete: true,
|
retainOnDelete: true,
|
||||||
ignoreChanges: ["clone", "started"],
|
ignoreChanges: ["clone", "started"],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ Bootstraps a k3s cluster on the Proxmox VMs created by `proxmox-infra`. Starts V
|
|||||||
2. Waits for port 22 to open on each VM (bash `/dev/tcp`)
|
2. Waits for port 22 to open on each VM (bash `/dev/tcp`)
|
||||||
3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
|
3. Installs k3s on `k3s-master-1` with `--cluster-init --tls-san <master1Ip>`
|
||||||
4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
|
4. Joins `k3s-master-2` and `k3s-master-3` as embedded etcd nodes
|
||||||
5. Joins `k3s-worker-1` and `k3s-worker-2` as agent nodes
|
5. Joins `k3s-worker-1`, `k3s-worker-2`, and `k3s-worker-3` as agent nodes
|
||||||
6. Formats the workers' dedicated `scsi1` disk (ext4, if not already formatted) and mounts it at `/mnt/longhorn-extra` (via `/etc/fstab`) — this is the disk `03-k8s-infra` later registers as an extra Longhorn disk, to keep Longhorn off the root filesystem and avoid `DiskPressure`
|
6. Formats the workers' dedicated `scsi1` disk (ext4, if not already formatted) and mounts it at `/mnt/longhorn-extra` (via `/etc/fstab`) — this is the disk `03-k8s-infra` later registers as an extra Longhorn disk, to keep Longhorn off the root filesystem and avoid `DiskPressure`
|
||||||
7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
|
7. Reads `/etc/rancher/k3s/k3s.yaml` from master-1 via SSH, patches the server URL, and exports it as the secret stack output `kubeconfig`
|
||||||
|
|
||||||
@@ -28,9 +28,10 @@ pulumi config set master2Ip "192.168.1.x"
|
|||||||
pulumi config set master3Ip "192.168.1.x"
|
pulumi config set master3Ip "192.168.1.x"
|
||||||
pulumi config set worker1Ip "192.168.1.x"
|
pulumi config set worker1Ip "192.168.1.x"
|
||||||
pulumi config set worker2Ip "192.168.1.x"
|
pulumi config set worker2Ip "192.168.1.x"
|
||||||
|
pulumi config set worker3Ip "192.168.1.x"
|
||||||
```
|
```
|
||||||
|
|
||||||
Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`) are read automatically from `proxmox-infra` via StackReference — do **not** set them here.
|
Proxmox credentials (`pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`, `pve3Endpoint`, `pve3ApiToken`) are read automatically from `proxmox-infra` via StackReference — do **not** set them here.
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
|
|||||||
@@ -3,12 +3,10 @@ import * as command from "@pulumi/command";
|
|||||||
|
|
||||||
const config = new pulumi.Config();
|
const config = new pulumi.Config();
|
||||||
|
|
||||||
//fetch credentials from proxmox-infra
|
|
||||||
const infraRef = new pulumi.StackReference(
|
const infraRef = new pulumi.StackReference(
|
||||||
`${pulumi.getOrganization()}/proxmox-infra/dev`,
|
`${pulumi.getOrganization()}/proxmox-infra/dev`,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Proxmox API credentials — same as proxmox-infra stack
|
|
||||||
const pve1Endpoint = infraRef.requireOutput(
|
const pve1Endpoint = infraRef.requireOutput(
|
||||||
"pve1Endpoint",
|
"pve1Endpoint",
|
||||||
) as pulumi.Output<string>;
|
) as pulumi.Output<string>;
|
||||||
@@ -21,6 +19,12 @@ const pve2Endpoint = infraRef.requireOutput(
|
|||||||
const pve2ApiToken = infraRef.requireOutput(
|
const pve2ApiToken = infraRef.requireOutput(
|
||||||
"pve2ApiToken",
|
"pve2ApiToken",
|
||||||
) as pulumi.Output<string>;
|
) as pulumi.Output<string>;
|
||||||
|
const pve3Endpoint = infraRef.requireOutput(
|
||||||
|
"pve3Endpoint",
|
||||||
|
) as pulumi.Output<string>;
|
||||||
|
const pve3ApiToken = infraRef.requireOutput(
|
||||||
|
"pve3ApiToken",
|
||||||
|
) as pulumi.Output<string>;
|
||||||
|
|
||||||
// Node IPs — static DHCP leases set in the router
|
// Node IPs — static DHCP leases set in the router
|
||||||
const master1Ip = infraRef.requireOutput("master1Ip");
|
const master1Ip = infraRef.requireOutput("master1Ip");
|
||||||
@@ -28,6 +32,7 @@ const master2Ip = infraRef.requireOutput("master2Ip");
|
|||||||
const master3Ip = infraRef.requireOutput("master3Ip");
|
const master3Ip = infraRef.requireOutput("master3Ip");
|
||||||
const worker1Ip = infraRef.requireOutput("worker1Ip");
|
const worker1Ip = infraRef.requireOutput("worker1Ip");
|
||||||
const worker2Ip = infraRef.requireOutput("worker2Ip");
|
const worker2Ip = infraRef.requireOutput("worker2Ip");
|
||||||
|
const worker3Ip = infraRef.requireOutput("worker3Ip");
|
||||||
|
|
||||||
// Pre-shared k3s cluster token
|
// Pre-shared k3s cluster token
|
||||||
const k3sToken = config.requireSecret("k3sToken");
|
const k3sToken = config.requireSecret("k3sToken");
|
||||||
@@ -45,6 +50,7 @@ const master2VmId = vmIdsOutput.apply((ids) => String(ids.master2));
|
|||||||
const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3));
|
const master3VmId = vmIdsOutput.apply((ids) => String(ids.master3));
|
||||||
const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1));
|
const worker1VmId = vmIdsOutput.apply((ids) => String(ids.worker1));
|
||||||
const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2));
|
const worker2VmId = vmIdsOutput.apply((ids) => String(ids.worker2));
|
||||||
|
const worker3VmId = vmIdsOutput.apply((ids) => String(ids.worker3));
|
||||||
|
|
||||||
// SSH connection helper
|
// SSH connection helper
|
||||||
function conn(
|
function conn(
|
||||||
@@ -83,6 +89,11 @@ const startWorker2 = new command.local.Command("start-worker-2", {
|
|||||||
triggers: [worker2VmId],
|
triggers: [worker2VmId],
|
||||||
interpreter: ["/bin/bash", "-c"],
|
interpreter: ["/bin/bash", "-c"],
|
||||||
});
|
});
|
||||||
|
const startWorker3 = new command.local.Command("start-worker-3", {
|
||||||
|
create: pulumi.interpolate`curl -sf -k -X POST -H "Authorization: PVEAPIToken=${pve3ApiToken}" "${pve3Endpoint}/api2/json/nodes/pve/qemu/${worker3VmId}/status/start" 2>/dev/null || true`,
|
||||||
|
triggers: [worker3VmId],
|
||||||
|
interpreter: ["/bin/bash", "-c"],
|
||||||
|
});
|
||||||
|
|
||||||
const allStarts = [
|
const allStarts = [
|
||||||
startMaster1,
|
startMaster1,
|
||||||
@@ -90,6 +101,7 @@ const allStarts = [
|
|||||||
startMaster3,
|
startMaster3,
|
||||||
startWorker1,
|
startWorker1,
|
||||||
startWorker2,
|
startWorker2,
|
||||||
|
startWorker3,
|
||||||
];
|
];
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -194,6 +206,16 @@ const waitWorker2Ssh = new command.local.Command(
|
|||||||
{ dependsOn: [joinMaster3] },
|
{ dependsOn: [joinMaster3] },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const waitWorker3Ssh = new command.local.Command(
|
||||||
|
"wait-ssh-worker-3",
|
||||||
|
{
|
||||||
|
create: pulumi.interpolate`for i in $(seq 1 60); do (timeout 5 bash -c "echo > /dev/tcp/${worker3Ip}/22") 2>/dev/null && exit 0; sleep 5; done; exit 1`,
|
||||||
|
triggers: [worker3VmId],
|
||||||
|
interpreter: ["/bin/bash", "-c"],
|
||||||
|
},
|
||||||
|
{ dependsOn: [joinMaster3] },
|
||||||
|
);
|
||||||
|
|
||||||
const joinWorker1 = new command.remote.Command(
|
const joinWorker1 = new command.remote.Command(
|
||||||
"join-k3s-worker-1",
|
"join-k3s-worker-1",
|
||||||
{
|
{
|
||||||
@@ -214,14 +236,24 @@ const joinWorker2 = new command.remote.Command(
|
|||||||
{ dependsOn: [waitWorker2Ssh] },
|
{ dependsOn: [waitWorker2Ssh] },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const joinWorker3 = new command.remote.Command(
|
||||||
|
"join-k3s-worker-3",
|
||||||
|
{
|
||||||
|
connection: conn(worker3Ip),
|
||||||
|
create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_URL=https://${master1Ip}:6443 K3S_TOKEN='${k3sToken}' sh -s - --node-name k3s-worker-3`,
|
||||||
|
triggers: [worker3VmId],
|
||||||
|
},
|
||||||
|
{ dependsOn: [waitWorker3Ssh] },
|
||||||
|
);
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Step 5: Format and mount Longhorn disks
|
// Step 5: Format and mount Longhorn disks
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const workers = [worker1Ip, worker2Ip];
|
const workers = [worker1Ip, worker2Ip, worker3Ip];
|
||||||
const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh];
|
const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh, waitWorker3Ssh];
|
||||||
|
|
||||||
const mountLonghornDisks = workers.map((worker, i) => {
|
workers.forEach((worker, i) => {
|
||||||
return new command.remote.Command(
|
return new command.remote.Command(
|
||||||
`mount-longhorn-disk-${i}`,
|
`mount-longhorn-disk-${i}`,
|
||||||
{
|
{
|
||||||
@@ -250,9 +282,64 @@ const getKubeconfig = new command.remote.Command(
|
|||||||
create: `sudo cat /etc/rancher/k3s/k3s.yaml`,
|
create: `sudo cat /etc/rancher/k3s/k3s.yaml`,
|
||||||
triggers: [master1VmId],
|
triggers: [master1VmId],
|
||||||
},
|
},
|
||||||
{ dependsOn: [joinWorker1, joinWorker2] },
|
{ dependsOn: [joinWorker1, joinWorker2, joinWorker3] },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Step 7: Install additional packages
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const allVms = [
|
||||||
|
{
|
||||||
|
id: master1VmId,
|
||||||
|
ip: master1Ip,
|
||||||
|
name: "master-1",
|
||||||
|
readiness: waitMaster1Ssh,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: master2VmId,
|
||||||
|
ip: master2Ip,
|
||||||
|
name: "master-2",
|
||||||
|
readiness: waitMaster2Ssh,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: master3VmId,
|
||||||
|
ip: master3Ip,
|
||||||
|
name: "master-3",
|
||||||
|
readiness: waitMaster3Ssh,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: worker1VmId,
|
||||||
|
ip: worker1Ip,
|
||||||
|
name: "worker-1",
|
||||||
|
readiness: waitWorker1Ssh,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: worker2VmId,
|
||||||
|
ip: worker2Ip,
|
||||||
|
name: "worker-2",
|
||||||
|
readiness: waitWorker2Ssh,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: worker3VmId,
|
||||||
|
ip: worker3Ip,
|
||||||
|
name: "worker-3",
|
||||||
|
readiness: waitWorker3Ssh,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
allVms.forEach(({ id, ip, name, readiness }) => {
|
||||||
|
new command.remote.Command(
|
||||||
|
`install-packages-${name}`,
|
||||||
|
{
|
||||||
|
connection: conn(ip),
|
||||||
|
create: pulumi.interpolate`sudo apt update && sudo apt install -y qemu-guest-agent && sudo systemctl enable --now qemu-guest-agent`,
|
||||||
|
triggers: [id],
|
||||||
|
},
|
||||||
|
{ dependsOn: readiness },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
export const kubeconfig = pulumi.secret(
|
export const kubeconfig = pulumi.secret(
|
||||||
pulumi
|
pulumi
|
||||||
.all([getKubeconfig.stdout, master1Ip])
|
.all([getKubeconfig.stdout, master1Ip])
|
||||||
|
|||||||
@@ -143,7 +143,7 @@ const longhorn = new k8s.helm.v3.Release(
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Patches Longhorn instances to use the mountend Longhorn storage on the worker nodes.
|
// Patches Longhorn instances to use the mountend Longhorn storage on the worker nodes.
|
||||||
["k3s-worker-1", "k3s-worker-2"].forEach(
|
["k3s-worker-1", "k3s-worker-2", "k3s-worker-3"].forEach(
|
||||||
(nodeName) =>
|
(nodeName) =>
|
||||||
new k8s.apiextensions.CustomResourcePatch(
|
new k8s.apiextensions.CustomResourcePatch(
|
||||||
`${nodeName}-longhorn-disks`,
|
`${nodeName}-longhorn-disks`,
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
# Git Conventions
|
||||||
|
|
||||||
|
## Branch Naming
|
||||||
|
|
||||||
|
```
|
||||||
|
<type>/<short-kebab-case-description>
|
||||||
|
```
|
||||||
|
|
||||||
|
- All lowercase, hyphens only — no camelCase or Title-Case
|
||||||
|
- Keep the description short (3–5 words); the branch name is not the place for detail
|
||||||
|
- No ticket tracker is used on this project, so branch names don't carry an ID
|
||||||
|
|
||||||
|
| Type | Use for |
|
||||||
|
|---|---|
|
||||||
|
| `feature/` | New functionality (e.g. a new stack, a new resource type) |
|
||||||
|
| `bug/` | Bug fixes |
|
||||||
|
| `enhancement/` | Improvements or refactors to existing functionality, no new capability |
|
||||||
|
| `chore/` | Dependency bumps, tooling, CI/CD, cleanup |
|
||||||
|
| `docs/` | Documentation only (README, CLAUDE.md files) |
|
||||||
|
|
||||||
|
**Examples:** `feature/add-3rd-proxmox-node`, `bug/fix-longhorn-disk-pressure`, `enhancement/refactor-k8s-bootstrap`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Commit Messages
|
||||||
|
|
||||||
|
```
|
||||||
|
<type>: <imperative, present-tense summary>
|
||||||
|
|
||||||
|
<optional body — explain WHY, not WHAT>
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Type** — same list as branch types above (`feature`, `bug`, `enhancement`, `chore`, `docs`)
|
||||||
|
- **Scope is optional** and, if used, should be one of the stack directories this repo is organized into — `proxmox-infra`, `k8s-bootstrap`, `k8s-infra`, `k8s-apps`, `monitoring` — or `ci` for `.gitea/workflows` changes: `fix(k8s-infra): ...`
|
||||||
|
- **Summary** — imperative mood ("add", not "added" or "adds"); no period at the end
|
||||||
|
- **Body** — only when the reasoning isn't obvious from the diff (a constraint, a workaround for a Proxmox/k3s quirk, a decision between StackReference outputs). Skip it for simple/self-explanatory changes.
|
||||||
|
|
||||||
|
**Examples:**
|
||||||
|
```
|
||||||
|
feature: add pfsense provider for automating static IP setup
|
||||||
|
bug: prune stale kube-vip leases after node replacement
|
||||||
|
chore: upgrade Pulumi provider packages across all stacks
|
||||||
|
docs: update CLAUDE.md with new Longhorn mount settings
|
||||||
|
enhancement(k8s-bootstrap): cache dependencies to speed up deployment
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Pull Request Titles
|
||||||
|
|
||||||
|
This repo uses Gitea Actions (see root `CLAUDE.md`); Gitea's merge commit reuses the PR title verbatim as the merge summary, so the title should stand on its own in `git log`. Frame it as the outcome, not a task log:
|
||||||
|
|
||||||
|
```
|
||||||
|
<Type>: <what changes for the cluster/infra>
|
||||||
|
```
|
||||||
|
|
||||||
|
**Examples:**
|
||||||
|
- `Feature: Add 3rd Proxmox node to the cluster`
|
||||||
|
- `Bug: Fix Longhorn disk pressure from root disk usage`
|
||||||
|
- `Enhancement: Refactor k8s-bootstrap for clearer deployment order`
|
||||||
|
- `Chore: Upgrade Pulumi packages across all stacks`
|
||||||
|
|
||||||
|
Since `01`/`02` deploys are manual (`workflow_dispatch`) and `03`–`05` deploy on merge to `main`, a clear PR title also doubles as a quick changelog of what just went live.
|
||||||
Reference in New Issue
Block a user