kasun/homelab-infrastructure-as-code
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

enhancement(proxmox-infra): consolidate per-host config into a single table
Deploy Proxmox Infra / Pulumi Preview (pull_request) Successful in 53s
Details
Deploy Proxmox Infra / Pulumi Deploy (pull_request) Skipped
Details
Deploy k8s Bootstrap / Pulumi Preview (pull_request) Successful in 40s
Details
Deploy k8s Bootstrap / Bootstrap k3s Cluster (pull_request) Skipped
Details

Browse Source 
Provider, image download, and template were three separate consts per
physical node with matching lookups scattered through nodeConfigs -
easy to update one and miss another when adding a node. Folded them
into one pveNodes pipeline (seed data -> provider -> image -> template)
and had nodeConfigs reference a host row instead of copying its fields.

Also folds in two fixes discovered while bringing up worker-3, now
inseparable from the restructuring above:
- disable the default physical CD-ROM passthrough (cdrom: fileId "none")
  that failed VM start on nodes without an optical drive
- shorten the qemu-agent timeout from the 15m default so refresh/read
  fails fast instead of hanging
This commit is contained in:
kasun
2026-07-30 02:55:01 +02:00
parent b7d5a5c5cd
commit 0ae688ac66
1 changed files with 127 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
01-proxmox-infra/index.ts
+122 -188
View File
@@ -25,33 +25,121 @@ const worker1Ip = config.requireSecret("worker1Ip");
const worker2Ip = config.requireSecret("worker2Ip");
const worker3Ip = config.requireSecret("worker3Ip");
// ---------------------------------------------------------------------------
// VM templates — one per node, cloned from the downloaded cloud image.
// Templates are not started and have no cloud-init config; that is applied
// per-clone so each node gets its own hostname (derived from VM name).
// ---------------------------------------------------------------------------
const templateSettings = {
template: true,
started: false,
stopOnDestroy: true,
scsiHardware: "virtio-scsi-pci",
cdrom: { fileId: "none" },
cpu: {
cores: 2,
sockets: 1,
type: "host",
numa: true,
},
memory: {
dedicated: 2048,
floating: 0,
},
networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
serialDevices: [{}],
vga: { type: "serial0" },
agent: { enabled: true, timeout: "30s" },
};
// ---------------------------------------------------------------------------
// Providers — one per standalone Proxmox machine
// ---------------------------------------------------------------------------
const pveProvider = new proxmox.Provider("pve", {
interface PveNodeConfig {
nodeName: string;
hostLabel: string;
isoStore: string;
dataStore: string;
endpoint: pulumi.Output<string>;
apiToken: pulumi.Output<string>;
}
const pveNodeSeeds: PveNodeConfig[] = [
{
nodeName: "pve",
hostLabel: "pve-b550",
isoStore: "pve-local-ext1",
dataStore: "local-lvm",
endpoint: pve1Endpoint,
apiToken: pve1ApiToken,
insecure: true,
});
const pveBckpProvider = new proxmox.Provider("pve-bckp", {
},
{
nodeName: "pve-bckp",
hostLabel: "pve-optiplex",
isoStore: "local",
dataStore: "local",
endpoint: pve2Endpoint,
apiToken: pve2ApiToken,
insecure: true,
});
const pveEsprimoProvider = new proxmox.Provider("pve-esprimo", {
},
{
nodeName: "pve",
hostLabel: "pve-esprimo",
isoStore: "local",
dataStore: "local-lvm",
endpoint: pve3Endpoint,
apiToken: pve3ApiToken,
insecure: true,
});
},
];
const pveNodes = {
"pve-b550": { nodeName: "pve", isoStore: "pve-local-ext1", dataStore: "local-lvm" },
"pve-optiplex": { nodeName: "pve-bckp", isoStore: "local", dataStore: "local" },
"pve-esprimo": { nodeName: "pve", isoStore: "local", dataStore: "local-lvm" },
} as const;
const pveNodes = pveNodeSeeds
.map((n) => ({
...n,
provider: new proxmox.Provider(n.hostLabel, {
endpoint: n.endpoint,
apiToken: n.apiToken,
insecure: true,
}),
}))
.map((n) => ({
...n,
ubuntuImagePve: new proxmox.download.File(
`ubuntu-noble-${n.hostLabel}`,
{
nodeName: n.nodeName,
datastoreId: n.isoStore,
contentType: "import",
fileName: "noble-server-cloudimg-amd64.qcow2",
url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img",
overwrite: false,
overwriteUnmanaged: true,
},
{ provider: n.provider },
),
}))
.map((n) => ({
...n,
pveTemplate: new proxmox.VmLegacy(
`k3s-template-${n.hostLabel}`,
{
...templateSettings,
nodeName: n.nodeName,
name: "k3s-ubuntu-noble-template",
disks: [
{
interface: "scsi0",
datastoreId: n.dataStore,
importFrom: pulumi.interpolate`${n.ubuntuImagePve.datastoreId}:import/${n.ubuntuImagePve.fileName}`,
size: 10,
ssd: true,
discard: "on",
},
],
},
{ provider: n.provider, ignoreChanges: ["disks"] },
),
}));
// ---------------------------------------------------------------------------
// Providers — PfSense
@@ -73,142 +161,6 @@ const ciRunnerKey = new tls.PrivateKey("ci-runner-key", {
algorithm: "ED25519",
});
// ---------------------------------------------------------------------------
// Download Ubuntu Noble cloud image to each node's ISO storage
// ---------------------------------------------------------------------------
const ubuntuNobleUrl =
"https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img";
const ubuntuImagePve = new proxmox.download.File(
"ubuntu-noble-pve",
{
nodeName: pveNodes["pve-b550"].nodeName,
datastoreId: pveNodes["pve-b550"].isoStore,
contentType: "import",
fileName: "noble-server-cloudimg-amd64.qcow2",
url: ubuntuNobleUrl,
overwrite: false,
overwriteUnmanaged: true,
},
{ provider: pveProvider },
);
const ubuntuImagePveBckp = new proxmox.download.File(
"ubuntu-noble-pve-bckp",
{
nodeName: pveNodes["pve-optiplex"].nodeName,
datastoreId: pveNodes["pve-optiplex"].isoStore,
contentType: "import",
fileName: "noble-server-cloudimg-amd64.qcow2",
url: ubuntuNobleUrl,
overwrite: false,
overwriteUnmanaged: true,
},
{ provider: pveBckpProvider },
);
const ubuntuImagePveEsprimo = new proxmox.download.File(
"ubuntu-noble-pve-esprimo",
{
nodeName: pveNodes["pve-esprimo"].nodeName,
datastoreId: pveNodes["pve-esprimo"].isoStore,
contentType: "import",
fileName: "noble-server-cloudimg-amd64.qcow2",
url: ubuntuNobleUrl,
overwrite: false,
overwriteUnmanaged: true,
},
{ provider: pveEsprimoProvider },
);
// ---------------------------------------------------------------------------
// VM templates — one per node, cloned from the downloaded cloud image.
// Templates are not started and have no cloud-init config; that is applied
// per-clone so each node gets its own hostname (derived from VM name).
// ---------------------------------------------------------------------------
const templateSettings = {
template: true,
started: false,
stopOnDestroy: true,
scsiHardware: "virtio-scsi-pci",
cpu: {
cores: 2,
sockets: 1,
type: "host",
numa: true,
},
memory: {
dedicated: 2048,
floating: 0,
},
networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
serialDevices: [{}],
vga: { type: "serial0" },
agent: { enabled: true },
};
const pveTemplate = new proxmox.VmLegacy(
"k3s-template-pve",
{
...templateSettings,
nodeName: "pve",
name: "k3s-ubuntu-noble-template",
disks: [
{
interface: "scsi0",
datastoreId: pveNodes["pve-b550"].dataStore,
importFrom: pulumi.interpolate`${ubuntuImagePve.datastoreId}:import/${ubuntuImagePve.fileName}`,
size: 10,
ssd: true,
discard: "on",
},
],
},
{ provider: pveProvider, ignoreChanges: ["disks"] },
);
const pveBckpTemplate = new proxmox.VmLegacy(
"k3s-template-pve-bckp",
{
...templateSettings,
nodeName: "pve-bckp",
name: "k3s-ubuntu-noble-template",
disks: [
{
interface: "scsi0",
datastoreId: pveNodes["pve-optiplex"].dataStore,
importFrom: pulumi.interpolate`${ubuntuImagePveBckp.datastoreId}:import/${ubuntuImagePveBckp.fileName}`,
size: 10,
ssd: true,
discard: "on",
},
],
},
{ provider: pveBckpProvider, ignoreChanges: ["disks"] },
);
const pveEsprimoTemplate = new proxmox.VmLegacy(
"k3s-template-pve-esprimo",
{
...templateSettings,
nodeName: "pve",
name: "k3s-ubuntu-noble-template",
disks: [
{
interface: "scsi0",
datastoreId: pveNodes["pve-esprimo"].dataStore,
importFrom: pulumi.interpolate`${ubuntuImagePveEsprimo.datastoreId}:import/${ubuntuImagePveEsprimo.fileName}`,
size: 10,
ssd: true,
discard: "on",
},
],
},
{ provider: pveEsprimoProvider, ignoreChanges: ["disks"] },
);
// ---------------------------------------------------------------------------
// k3s nodes — full clones of their respective templates
// ---------------------------------------------------------------------------
@@ -219,69 +171,50 @@ const sshPvePublicKey = config.requireSecret("sshPvePublicKey");
interface NodeConfig {
name: string;
role: "master" | "worker";
nodeName: string;
provider: proxmox.Provider;
template: proxmox.VmLegacy;
diskDatastore: string;
host: (typeof pveNodes)[number]; // one row from the pveNodes chain
ip: pulumi.Output<string>;
longhornDiskSize?: number;
}
const hostFor = (label: string) => pveNodes.find((n) => n.hostLabel === label)!;
const nodeConfigs: NodeConfig[] = [
{
name: "k3s-master-1",
role: "master",
nodeName: pveNodes["pve-b550"].nodeName,
provider: pveProvider,
template: pveTemplate,
diskDatastore: pveNodes["pve-b550"].dataStore,
host: hostFor("pve-b550"),
ip: master1Ip,
},
{
name: "k3s-master-2",
role: "master",
nodeName: pveNodes["pve-esprimo"].nodeName,
provider: pveEsprimoProvider,
template: pveTemplate,
diskDatastore: pveNodes["pve-esprimo"].dataStore,
host: hostFor("pve-esprimo"),
ip: master2Ip,
},
{
name: "k3s-worker-1",
role: "worker",
nodeName: pveNodes["pve-b550"].nodeName,
provider: pveProvider,
template: pveTemplate,
diskDatastore: pveNodes["pve-b550"].dataStore,
host: hostFor("pve-b550"),
ip: worker1Ip,
longhornDiskSize: 50,
},
{
name: "k3s-master-3",
role: "master",
nodeName: pveNodes["pve-optiplex"].nodeName,
provider: pveBckpProvider,
template: pveBckpTemplate,
diskDatastore: pveNodes["pve-optiplex"].dataStore,
host: hostFor("pve-optiplex"),
ip: master3Ip,
},
{
name: "k3s-worker-2",
role: "worker",
nodeName: pveNodes["pve-optiplex"].nodeName,
provider: pveBckpProvider,
template: pveBckpTemplate,
diskDatastore: pveNodes["pve-optiplex"].dataStore,
host: hostFor("pve-optiplex"),
ip: worker2Ip,
longhornDiskSize: 50,
},
{
name: "k3s-worker-3",
role: "worker",
nodeName: pveNodes["pve-esprimo"].nodeName,
provider: pveEsprimoProvider,
template: pveEsprimoTemplate,
diskDatastore: pveNodes["pve-esprimo"].dataStore,
host: hostFor("pve-esprimo"),
ip: worker3Ip,
longhornDiskSize: 50,
},
@@ -292,14 +225,14 @@ const k3sVms = nodeConfigs.map(
new proxmox.VmLegacy(
node.name,
{
nodeName: node.nodeName,
nodeName: node.host.nodeName,
name: node.name,
description: "k3s " + node.role + " node — managed by Pulumi",
tags: ["k3s", node.role],
clone: {
vmId: node.template.vmId,
vmId: node.host.pveTemplate.vmId,
full: true,
datastoreId: node.diskDatastore,
datastoreId: node.host.dataStore,
},
cpu: {
cores: 2,
@@ -314,7 +247,7 @@ const k3sVms = nodeConfigs.map(
disks: [
{
interface: "scsi0",
datastoreId: node.diskDatastore,
datastoreId: node.host.dataStore,
size: 35,
ssd: true,
discard: "on",
@@ -324,7 +257,7 @@ const k3sVms = nodeConfigs.map(
: [
{
interface: "scsi1",
datastoreId: node.diskDatastore,
datastoreId: node.host.dataStore,
size: node.longhornDiskSize,
ssd: true,
discard: "on",
@@ -332,7 +265,7 @@ const k3sVms = nodeConfigs.map(
]),
],
initialization: {
datastoreId: node.diskDatastore,
datastoreId: node.host.dataStore,
ipConfigs: [{ ipv4: { address: "dhcp" } }],
userAccount: {
username: "ubuntu",
@@ -347,12 +280,13 @@ const k3sVms = nodeConfigs.map(
scsiHardware: "virtio-scsi-pci",
serialDevices: [{}],
vga: { type: "serial0" },
agent: { enabled: true },
agent: { enabled: true, timeout: "30s" },
cdrom: { fileId: "none" },
started: false,
stopOnDestroy: true,
},
{
provider: node.provider,
provider: node.host.provider,
retainOnDelete: true,
ignoreChanges: ["clone", "started"],
},