Merge pull request 'Enhancement/consolidate proxmox node config' (#32) from enhancement/consolidate-proxmox-node-config into main

Reviewed-on: #32

01-proxmox-infra/index.ts

@@ -25,33 +25,121 @@ const worker1Ip = config.requireSecret("worker1Ip");
 const worker2Ip = config.requireSecret("worker2Ip");
 const worker3Ip = config.requireSecret("worker3Ip");
 
+// ---------------------------------------------------------------------------
+// VM templates — one per node, cloned from the downloaded cloud image.
+// Templates are not started and have no cloud-init config; that is applied
+// per-clone so each node gets its own hostname (derived from VM name).
+// ---------------------------------------------------------------------------
+
+const templateSettings = {
+  template: true,
+  started: false,
+  stopOnDestroy: true,
+  scsiHardware: "virtio-scsi-pci",
+  cdrom: { fileId: "none" },
+  cpu: {
+    cores: 2,
+    sockets: 1,
+    type: "host",
+    numa: true,
+  },
+  memory: {
+    dedicated: 2048,
+    floating: 0,
+  },
+  networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
+  serialDevices: [{}],
+  vga: { type: "serial0" },
+  agent: { enabled: true, timeout: "30s" },
+};
+
 // ---------------------------------------------------------------------------
 // Providers — one per standalone Proxmox machine
 // ---------------------------------------------------------------------------
 
-const pveProvider = new proxmox.Provider("pve", {
+interface PveNodeConfig {
-  endpoint: pve1Endpoint,
+  nodeName: string;
-  apiToken: pve1ApiToken,
+  hostLabel: string;
-  insecure: true,
+  isoStore: string;
-});
+  dataStore: string;
+  endpoint: pulumi.Output<string>;
+  apiToken: pulumi.Output<string>;
+}
 
-const pveBckpProvider = new proxmox.Provider("pve-bckp", {
+const pveNodeSeeds: PveNodeConfig[] = [
-  endpoint: pve2Endpoint,
+  {
-  apiToken: pve2ApiToken,
+    nodeName: "pve",
-  insecure: true,
+    hostLabel: "pve-b550",
-});
+    isoStore: "pve-local-ext1",
+    dataStore: "local-lvm",
+    endpoint: pve1Endpoint,
+    apiToken: pve1ApiToken,
+  },
+  {
+    nodeName: "pve-bckp",
+    hostLabel: "pve-optiplex",
+    isoStore: "local",
+    dataStore: "local",
+    endpoint: pve2Endpoint,
+    apiToken: pve2ApiToken,
+  },
+  {
+    nodeName: "pve",
+    hostLabel: "pve-esprimo",
+    isoStore: "local",
+    dataStore: "local-lvm",
+    endpoint: pve3Endpoint,
+    apiToken: pve3ApiToken,
+  },
+];
 
-const pveEsprimoProvider = new proxmox.Provider("pve-esprimo", {
+const pveNodes = pveNodeSeeds
-  endpoint: pve3Endpoint,
+  .map((n) => ({
-  apiToken: pve3ApiToken,
+    ...n,
-  insecure: true,
+    provider: new proxmox.Provider(n.hostLabel, {
-});
+      endpoint: n.endpoint,
-
+      apiToken: n.apiToken,
-const pveNodes = {
+      insecure: true,
-  "pve-b550": { nodeName: "pve", isoStore: "pve-local-ext1", dataStore: "local-lvm" },
+    }),
-  "pve-optiplex": { nodeName: "pve-bckp", isoStore: "local", dataStore: "local" },
+  }))
-  "pve-esprimo": { nodeName: "pve", isoStore: "local", dataStore: "local-lvm" },
+  .map((n) => ({
-} as const;
+    ...n,
+    ubuntuImagePve: new proxmox.download.File(
+      `ubuntu-noble-${n.hostLabel}`,
+      {
+        nodeName: n.nodeName,
+        datastoreId: n.isoStore,
+        contentType: "import",
+        fileName: "noble-server-cloudimg-amd64.qcow2",
+        url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img",
+        overwrite: false,
+        overwriteUnmanaged: true,
+      },
+      { provider: n.provider },
+    ),
+  }))
+  .map((n) => ({
+    ...n,
+    pveTemplate: new proxmox.VmLegacy(
+      `k3s-template-${n.hostLabel}`,
+      {
+        ...templateSettings,
+        nodeName: n.nodeName,
+        name: "k3s-ubuntu-noble-template",
+        disks: [
+          {
+            interface: "scsi0",
+            datastoreId: n.dataStore,
+            importFrom: pulumi.interpolate`${n.ubuntuImagePve.datastoreId}:import/${n.ubuntuImagePve.fileName}`,
+            size: 10,
+            ssd: true,
+            discard: "on",
+          },
+        ],
+      },
+      { provider: n.provider, ignoreChanges: ["disks"] },
+    ),
+  }));
 
 // ---------------------------------------------------------------------------
 // Providers — PfSense
@@ -73,215 +161,62 @@ const ciRunnerKey = new tls.PrivateKey("ci-runner-key", {
   algorithm: "ED25519",
 });
 
-// ---------------------------------------------------------------------------
-// Download Ubuntu Noble cloud image to each node's ISO storage
-// ---------------------------------------------------------------------------
-
-const ubuntuNobleUrl =
-  "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img";
-
-const ubuntuImagePve = new proxmox.download.File(
-  "ubuntu-noble-pve",
-  {
-    nodeName: pveNodes["pve-b550"].nodeName,
-    datastoreId: pveNodes["pve-b550"].isoStore,
-    contentType: "import",
-    fileName: "noble-server-cloudimg-amd64.qcow2",
-    url: ubuntuNobleUrl,
-    overwrite: false,
-    overwriteUnmanaged: true,
-  },
-  { provider: pveProvider },
-);
-
-const ubuntuImagePveBckp = new proxmox.download.File(
-  "ubuntu-noble-pve-bckp",
-  {
-    nodeName: pveNodes["pve-optiplex"].nodeName,
-    datastoreId: pveNodes["pve-optiplex"].isoStore,
-    contentType: "import",
-    fileName: "noble-server-cloudimg-amd64.qcow2",
-    url: ubuntuNobleUrl,
-    overwrite: false,
-    overwriteUnmanaged: true,
-  },
-  { provider: pveBckpProvider },
-);
-
-const ubuntuImagePveEsprimo = new proxmox.download.File(
-  "ubuntu-noble-pve-esprimo",
-  {
-    nodeName: pveNodes["pve-esprimo"].nodeName,
-    datastoreId: pveNodes["pve-esprimo"].isoStore,
-    contentType: "import",
-    fileName: "noble-server-cloudimg-amd64.qcow2",
-    url: ubuntuNobleUrl,
-    overwrite: false,
-    overwriteUnmanaged: true,
-  },
-  { provider: pveEsprimoProvider },
-);
-
-// ---------------------------------------------------------------------------
-// VM templates — one per node, cloned from the downloaded cloud image.
-// Templates are not started and have no cloud-init config; that is applied
-// per-clone so each node gets its own hostname (derived from VM name).
-// ---------------------------------------------------------------------------
-
-const templateSettings = {
-  template: true,
-  started: false,
-  stopOnDestroy: true,
-  scsiHardware: "virtio-scsi-pci",
-  cpu: {
-    cores: 2,
-    sockets: 1,
-    type: "host",
-    numa: true,
-  },
-  memory: {
-    dedicated: 2048,
-    floating: 0,
-  },
-  networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
-  serialDevices: [{}],
-  vga: { type: "serial0" },
-  agent: { enabled: true },
-};
-
-const pveTemplate = new proxmox.VmLegacy(
-  "k3s-template-pve",
-  {
-    ...templateSettings,
-    nodeName: "pve",
-    name: "k3s-ubuntu-noble-template",
-    disks: [
-      {
-        interface: "scsi0",
-        datastoreId: pveNodes["pve-b550"].dataStore,
-        importFrom: pulumi.interpolate`${ubuntuImagePve.datastoreId}:import/${ubuntuImagePve.fileName}`,
-        size: 10,
-        ssd: true,
-        discard: "on",
-      },
-    ],
-  },
-  { provider: pveProvider, ignoreChanges: ["disks"] },
-);
-
-const pveBckpTemplate = new proxmox.VmLegacy(
-  "k3s-template-pve-bckp",
-  {
-    ...templateSettings,
-    nodeName: "pve-bckp",
-    name: "k3s-ubuntu-noble-template",
-    disks: [
-      {
-        interface: "scsi0",
-        datastoreId: pveNodes["pve-optiplex"].dataStore,
-        importFrom: pulumi.interpolate`${ubuntuImagePveBckp.datastoreId}:import/${ubuntuImagePveBckp.fileName}`,
-        size: 10,
-        ssd: true,
-        discard: "on",
-      },
-    ],
-  },
-  { provider: pveBckpProvider, ignoreChanges: ["disks"] },
-);
-
-const pveEsprimoTemplate = new proxmox.VmLegacy(
-  "k3s-template-pve-esprimo",
-  {
-    ...templateSettings,
-    nodeName: "pve",
-    name: "k3s-ubuntu-noble-template",
-    disks: [
-      {
-        interface: "scsi0",
-        datastoreId: pveNodes["pve-esprimo"].dataStore,
-        importFrom: pulumi.interpolate`${ubuntuImagePveEsprimo.datastoreId}:import/${ubuntuImagePveEsprimo.fileName}`,
-        size: 10,
-        ssd: true,
-        discard: "on",
-      },
-    ],
-  },
-  { provider: pveEsprimoProvider, ignoreChanges: ["disks"] },
-);
-
 // ---------------------------------------------------------------------------
 // k3s nodes — full clones of their respective templates
 // ---------------------------------------------------------------------------
 
 const k3sVmPassword = config.requireSecret("k3sVmPassword");
-const sshPvePublicKey = config.requireSecret("sshPvePublicKey");
+const pve1SshPublicKey = config.requireSecret("pve1SshPublicKey");
+const pve2SshPublicKey = config.requireSecret("pve2SshPublicKey");
+const pve3SshPublicKey = config.requireSecret("pve3SshPublicKey");
 
 interface NodeConfig {
   name: string;
   role: "master" | "worker";
-  nodeName: string;
+  host: (typeof pveNodes)[number]; // one row from the pveNodes chain
-  provider: proxmox.Provider;
-  template: proxmox.VmLegacy;
-  diskDatastore: string;
   ip: pulumi.Output<string>;
   longhornDiskSize?: number;
 }
 
+const hostFor = (label: string) => pveNodes.find((n) => n.hostLabel === label)!;
+
 const nodeConfigs: NodeConfig[] = [
   {
     name: "k3s-master-1",
     role: "master",
-    nodeName: pveNodes["pve-b550"].nodeName,
+    host: hostFor("pve-b550"),
-    provider: pveProvider,
-    template: pveTemplate,
-    diskDatastore: pveNodes["pve-b550"].dataStore,
     ip: master1Ip,
   },
   {
     name: "k3s-master-2",
     role: "master",
-    nodeName: pveNodes["pve-esprimo"].nodeName,
+    host: hostFor("pve-esprimo"),
-    provider: pveEsprimoProvider,
-    template: pveTemplate,
-    diskDatastore: pveNodes["pve-esprimo"].dataStore,
     ip: master2Ip,
   },
   {
     name: "k3s-worker-1",
     role: "worker",
-    nodeName: pveNodes["pve-b550"].nodeName,
+    host: hostFor("pve-b550"),
-    provider: pveProvider,
-    template: pveTemplate,
-    diskDatastore: pveNodes["pve-b550"].dataStore,
     ip: worker1Ip,
     longhornDiskSize: 50,
   },
   {
     name: "k3s-master-3",
     role: "master",
-    nodeName: pveNodes["pve-optiplex"].nodeName,
+    host: hostFor("pve-optiplex"),
-    provider: pveBckpProvider,
-    template: pveBckpTemplate,
-    diskDatastore: pveNodes["pve-optiplex"].dataStore,
     ip: master3Ip,
   },
   {
     name: "k3s-worker-2",
     role: "worker",
-    nodeName: pveNodes["pve-optiplex"].nodeName,
+    host: hostFor("pve-optiplex"),
-    provider: pveBckpProvider,
-    template: pveBckpTemplate,
-    diskDatastore: pveNodes["pve-optiplex"].dataStore,
     ip: worker2Ip,
     longhornDiskSize: 50,
   },
   {
     name: "k3s-worker-3",
     role: "worker",
-    nodeName: pveNodes["pve-esprimo"].nodeName,
+    host: hostFor("pve-esprimo"),
-    provider: pveEsprimoProvider,
-    template: pveEsprimoTemplate,
-    diskDatastore: pveNodes["pve-esprimo"].dataStore,
     ip: worker3Ip,
     longhornDiskSize: 50,
   },
@@ -292,14 +227,14 @@ const k3sVms = nodeConfigs.map(
     new proxmox.VmLegacy(
       node.name,
       {
-        nodeName: node.nodeName,
+        nodeName: node.host.nodeName,
         name: node.name,
         description: "k3s " + node.role + " node — managed by Pulumi",
         tags: ["k3s", node.role],
         clone: {
-          vmId: node.template.vmId,
+          vmId: node.host.pveTemplate.vmId,
           full: true,
-          datastoreId: node.diskDatastore,
+          datastoreId: node.host.dataStore,
         },
         cpu: {
           cores: 2,
@@ -314,7 +249,7 @@ const k3sVms = nodeConfigs.map(
         disks: [
           {
             interface: "scsi0",
-            datastoreId: node.diskDatastore,
+            datastoreId: node.host.dataStore,
             size: 35,
             ssd: true,
             discard: "on",
@@ -324,7 +259,7 @@ const k3sVms = nodeConfigs.map(
             : [
                 {
                   interface: "scsi1",
-                  datastoreId: node.diskDatastore,
+                  datastoreId: node.host.dataStore,
                   size: node.longhornDiskSize,
                   ssd: true,
                   discard: "on",
@@ -332,13 +267,15 @@ const k3sVms = nodeConfigs.map(
               ]),
         ],
         initialization: {
-          datastoreId: node.diskDatastore,
+          datastoreId: node.host.dataStore,
           ipConfigs: [{ ipv4: { address: "dhcp" } }],
           userAccount: {
             username: "ubuntu",
             password: k3sVmPassword,
             keys: [
-              sshPvePublicKey.apply((k) => k.trim()),
+              pve1SshPublicKey.apply((k) => k.trim()),
+              pve2SshPublicKey.apply((k) => k.trim()),
+              pve3SshPublicKey.apply((k) => k.trim()),
               ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
             ],
           },
@@ -347,12 +284,13 @@ const k3sVms = nodeConfigs.map(
         scsiHardware: "virtio-scsi-pci",
         serialDevices: [{}],
         vga: { type: "serial0" },
-        agent: { enabled: true },
+        agent: { enabled: true, timeout: "30s" },
+        cdrom: { fileId: "none" },
         started: false,
         stopOnDestroy: true,
       },
       {
-        provider: node.provider,
+        provider: node.host.provider,
         retainOnDelete: true,
         ignoreChanges: ["clone", "started"],
       },

02-k8s-bootstrap/index.ts

@@ -3,12 +3,10 @@ import * as command from "@pulumi/command";
 
 const config = new pulumi.Config();
 
-//fetch credentials from proxmox-infra
 const infraRef = new pulumi.StackReference(
   `${pulumi.getOrganization()}/proxmox-infra/dev`,
 );
 
-// Proxmox API credentials — same as proxmox-infra stack
 const pve1Endpoint = infraRef.requireOutput(
   "pve1Endpoint",
 ) as pulumi.Output<string>;
@@ -255,7 +253,7 @@ const joinWorker3 = new command.remote.Command(
 const workers = [worker1Ip, worker2Ip, worker3Ip];
 const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh, waitWorker3Ssh];
 
-const mountLonghornDisks = workers.map((worker, i) => {
+workers.forEach((worker, i) => {
   return new command.remote.Command(
     `mount-longhorn-disk-${i}`,
     {
@@ -287,6 +285,61 @@ const getKubeconfig = new command.remote.Command(
   { dependsOn: [joinWorker1, joinWorker2, joinWorker3] },
 );
 
+// ---------------------------------------------------------------------------
+// Step 7: Install additional packages
+// ---------------------------------------------------------------------------
+
+const allVms = [
+  {
+    id: master1VmId,
+    ip: master1Ip,
+    name: "master-1",
+    readiness: waitMaster1Ssh,
+  },
+  {
+    id: master2VmId,
+    ip: master2Ip,
+    name: "master-2",
+    readiness: waitMaster2Ssh,
+  },
+  {
+    id: master3VmId,
+    ip: master3Ip,
+    name: "master-3",
+    readiness: waitMaster3Ssh,
+  },
+  {
+    id: worker1VmId,
+    ip: worker1Ip,
+    name: "worker-1",
+    readiness: waitWorker1Ssh,
+  },
+  {
+    id: worker2VmId,
+    ip: worker2Ip,
+    name: "worker-2",
+    readiness: waitWorker2Ssh,
+  },
+  {
+    id: worker3VmId,
+    ip: worker3Ip,
+    name: "worker-3",
+    readiness: waitWorker3Ssh,
+  },
+];
+
+allVms.forEach(({ id, ip, name, readiness }) => {
+  new command.remote.Command(
+    `install-packages-${name}`,
+    {
+      connection: conn(ip),
+      create: pulumi.interpolate`sudo apt update && sudo apt install -y qemu-guest-agent && sudo systemctl enable --now qemu-guest-agent`,
+      triggers: [id],
+    },
+    { dependsOn: readiness },
+  );
+});
+
 export const kubeconfig = pulumi.secret(
   pulumi
     .all([getKubeconfig.stdout, master1Ip])