Merge pull request 'Enhancement/consolidate proxmox node config' (#32) from enhancement/consolidate-proxmox-node-config into main
Reviewed-on: #32
This commit was merged in pull request #32.
This commit is contained in:
+128
-190
@@ -25,33 +25,121 @@ const worker1Ip = config.requireSecret("worker1Ip");
|
||||
const worker2Ip = config.requireSecret("worker2Ip");
|
||||
const worker3Ip = config.requireSecret("worker3Ip");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// VM templates — one per node, cloned from the downloaded cloud image.
|
||||
// Templates are not started and have no cloud-init config; that is applied
|
||||
// per-clone so each node gets its own hostname (derived from VM name).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const templateSettings = {
|
||||
template: true,
|
||||
started: false,
|
||||
stopOnDestroy: true,
|
||||
scsiHardware: "virtio-scsi-pci",
|
||||
cdrom: { fileId: "none" },
|
||||
cpu: {
|
||||
cores: 2,
|
||||
sockets: 1,
|
||||
type: "host",
|
||||
numa: true,
|
||||
},
|
||||
memory: {
|
||||
dedicated: 2048,
|
||||
floating: 0,
|
||||
},
|
||||
networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
|
||||
serialDevices: [{}],
|
||||
vga: { type: "serial0" },
|
||||
agent: { enabled: true, timeout: "30s" },
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Providers — one per standalone Proxmox machine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const pveProvider = new proxmox.Provider("pve", {
|
||||
interface PveNodeConfig {
|
||||
nodeName: string;
|
||||
hostLabel: string;
|
||||
isoStore: string;
|
||||
dataStore: string;
|
||||
endpoint: pulumi.Output<string>;
|
||||
apiToken: pulumi.Output<string>;
|
||||
}
|
||||
|
||||
const pveNodeSeeds: PveNodeConfig[] = [
|
||||
{
|
||||
nodeName: "pve",
|
||||
hostLabel: "pve-b550",
|
||||
isoStore: "pve-local-ext1",
|
||||
dataStore: "local-lvm",
|
||||
endpoint: pve1Endpoint,
|
||||
apiToken: pve1ApiToken,
|
||||
insecure: true,
|
||||
});
|
||||
|
||||
const pveBckpProvider = new proxmox.Provider("pve-bckp", {
|
||||
},
|
||||
{
|
||||
nodeName: "pve-bckp",
|
||||
hostLabel: "pve-optiplex",
|
||||
isoStore: "local",
|
||||
dataStore: "local",
|
||||
endpoint: pve2Endpoint,
|
||||
apiToken: pve2ApiToken,
|
||||
insecure: true,
|
||||
});
|
||||
|
||||
const pveEsprimoProvider = new proxmox.Provider("pve-esprimo", {
|
||||
},
|
||||
{
|
||||
nodeName: "pve",
|
||||
hostLabel: "pve-esprimo",
|
||||
isoStore: "local",
|
||||
dataStore: "local-lvm",
|
||||
endpoint: pve3Endpoint,
|
||||
apiToken: pve3ApiToken,
|
||||
insecure: true,
|
||||
});
|
||||
},
|
||||
];
|
||||
|
||||
const pveNodes = {
|
||||
"pve-b550": { nodeName: "pve", isoStore: "pve-local-ext1", dataStore: "local-lvm" },
|
||||
"pve-optiplex": { nodeName: "pve-bckp", isoStore: "local", dataStore: "local" },
|
||||
"pve-esprimo": { nodeName: "pve", isoStore: "local", dataStore: "local-lvm" },
|
||||
} as const;
|
||||
const pveNodes = pveNodeSeeds
|
||||
.map((n) => ({
|
||||
...n,
|
||||
provider: new proxmox.Provider(n.hostLabel, {
|
||||
endpoint: n.endpoint,
|
||||
apiToken: n.apiToken,
|
||||
insecure: true,
|
||||
}),
|
||||
}))
|
||||
.map((n) => ({
|
||||
...n,
|
||||
ubuntuImagePve: new proxmox.download.File(
|
||||
`ubuntu-noble-${n.hostLabel}`,
|
||||
{
|
||||
nodeName: n.nodeName,
|
||||
datastoreId: n.isoStore,
|
||||
contentType: "import",
|
||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
||||
url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img",
|
||||
overwrite: false,
|
||||
overwriteUnmanaged: true,
|
||||
},
|
||||
{ provider: n.provider },
|
||||
),
|
||||
}))
|
||||
.map((n) => ({
|
||||
...n,
|
||||
pveTemplate: new proxmox.VmLegacy(
|
||||
`k3s-template-${n.hostLabel}`,
|
||||
{
|
||||
...templateSettings,
|
||||
nodeName: n.nodeName,
|
||||
name: "k3s-ubuntu-noble-template",
|
||||
disks: [
|
||||
{
|
||||
interface: "scsi0",
|
||||
datastoreId: n.dataStore,
|
||||
importFrom: pulumi.interpolate`${n.ubuntuImagePve.datastoreId}:import/${n.ubuntuImagePve.fileName}`,
|
||||
size: 10,
|
||||
ssd: true,
|
||||
discard: "on",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ provider: n.provider, ignoreChanges: ["disks"] },
|
||||
),
|
||||
}));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Providers — PfSense
|
||||
@@ -73,215 +161,62 @@ const ciRunnerKey = new tls.PrivateKey("ci-runner-key", {
|
||||
algorithm: "ED25519",
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Download Ubuntu Noble cloud image to each node's ISO storage
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ubuntuNobleUrl =
|
||||
"https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img";
|
||||
|
||||
const ubuntuImagePve = new proxmox.download.File(
|
||||
"ubuntu-noble-pve",
|
||||
{
|
||||
nodeName: pveNodes["pve-b550"].nodeName,
|
||||
datastoreId: pveNodes["pve-b550"].isoStore,
|
||||
contentType: "import",
|
||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
||||
url: ubuntuNobleUrl,
|
||||
overwrite: false,
|
||||
overwriteUnmanaged: true,
|
||||
},
|
||||
{ provider: pveProvider },
|
||||
);
|
||||
|
||||
const ubuntuImagePveBckp = new proxmox.download.File(
|
||||
"ubuntu-noble-pve-bckp",
|
||||
{
|
||||
nodeName: pveNodes["pve-optiplex"].nodeName,
|
||||
datastoreId: pveNodes["pve-optiplex"].isoStore,
|
||||
contentType: "import",
|
||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
||||
url: ubuntuNobleUrl,
|
||||
overwrite: false,
|
||||
overwriteUnmanaged: true,
|
||||
},
|
||||
{ provider: pveBckpProvider },
|
||||
);
|
||||
|
||||
const ubuntuImagePveEsprimo = new proxmox.download.File(
|
||||
"ubuntu-noble-pve-esprimo",
|
||||
{
|
||||
nodeName: pveNodes["pve-esprimo"].nodeName,
|
||||
datastoreId: pveNodes["pve-esprimo"].isoStore,
|
||||
contentType: "import",
|
||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
||||
url: ubuntuNobleUrl,
|
||||
overwrite: false,
|
||||
overwriteUnmanaged: true,
|
||||
},
|
||||
{ provider: pveEsprimoProvider },
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// VM templates — one per node, cloned from the downloaded cloud image.
|
||||
// Templates are not started and have no cloud-init config; that is applied
|
||||
// per-clone so each node gets its own hostname (derived from VM name).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const templateSettings = {
|
||||
template: true,
|
||||
started: false,
|
||||
stopOnDestroy: true,
|
||||
scsiHardware: "virtio-scsi-pci",
|
||||
cpu: {
|
||||
cores: 2,
|
||||
sockets: 1,
|
||||
type: "host",
|
||||
numa: true,
|
||||
},
|
||||
memory: {
|
||||
dedicated: 2048,
|
||||
floating: 0,
|
||||
},
|
||||
networkDevices: [{ bridge: "vmbr0", model: "virtio" }],
|
||||
serialDevices: [{}],
|
||||
vga: { type: "serial0" },
|
||||
agent: { enabled: true },
|
||||
};
|
||||
|
||||
const pveTemplate = new proxmox.VmLegacy(
|
||||
"k3s-template-pve",
|
||||
{
|
||||
...templateSettings,
|
||||
nodeName: "pve",
|
||||
name: "k3s-ubuntu-noble-template",
|
||||
disks: [
|
||||
{
|
||||
interface: "scsi0",
|
||||
datastoreId: pveNodes["pve-b550"].dataStore,
|
||||
importFrom: pulumi.interpolate`${ubuntuImagePve.datastoreId}:import/${ubuntuImagePve.fileName}`,
|
||||
size: 10,
|
||||
ssd: true,
|
||||
discard: "on",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ provider: pveProvider, ignoreChanges: ["disks"] },
|
||||
);
|
||||
|
||||
const pveBckpTemplate = new proxmox.VmLegacy(
|
||||
"k3s-template-pve-bckp",
|
||||
{
|
||||
...templateSettings,
|
||||
nodeName: "pve-bckp",
|
||||
name: "k3s-ubuntu-noble-template",
|
||||
disks: [
|
||||
{
|
||||
interface: "scsi0",
|
||||
datastoreId: pveNodes["pve-optiplex"].dataStore,
|
||||
importFrom: pulumi.interpolate`${ubuntuImagePveBckp.datastoreId}:import/${ubuntuImagePveBckp.fileName}`,
|
||||
size: 10,
|
||||
ssd: true,
|
||||
discard: "on",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ provider: pveBckpProvider, ignoreChanges: ["disks"] },
|
||||
);
|
||||
|
||||
const pveEsprimoTemplate = new proxmox.VmLegacy(
|
||||
"k3s-template-pve-esprimo",
|
||||
{
|
||||
...templateSettings,
|
||||
nodeName: "pve",
|
||||
name: "k3s-ubuntu-noble-template",
|
||||
disks: [
|
||||
{
|
||||
interface: "scsi0",
|
||||
datastoreId: pveNodes["pve-esprimo"].dataStore,
|
||||
importFrom: pulumi.interpolate`${ubuntuImagePveEsprimo.datastoreId}:import/${ubuntuImagePveEsprimo.fileName}`,
|
||||
size: 10,
|
||||
ssd: true,
|
||||
discard: "on",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ provider: pveEsprimoProvider, ignoreChanges: ["disks"] },
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// k3s nodes — full clones of their respective templates
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const k3sVmPassword = config.requireSecret("k3sVmPassword");
|
||||
const sshPvePublicKey = config.requireSecret("sshPvePublicKey");
|
||||
const pve1SshPublicKey = config.requireSecret("pve1SshPublicKey");
|
||||
const pve2SshPublicKey = config.requireSecret("pve2SshPublicKey");
|
||||
const pve3SshPublicKey = config.requireSecret("pve3SshPublicKey");
|
||||
|
||||
interface NodeConfig {
|
||||
name: string;
|
||||
role: "master" | "worker";
|
||||
nodeName: string;
|
||||
provider: proxmox.Provider;
|
||||
template: proxmox.VmLegacy;
|
||||
diskDatastore: string;
|
||||
host: (typeof pveNodes)[number]; // one row from the pveNodes chain
|
||||
ip: pulumi.Output<string>;
|
||||
longhornDiskSize?: number;
|
||||
}
|
||||
|
||||
const hostFor = (label: string) => pveNodes.find((n) => n.hostLabel === label)!;
|
||||
|
||||
const nodeConfigs: NodeConfig[] = [
|
||||
{
|
||||
name: "k3s-master-1",
|
||||
role: "master",
|
||||
nodeName: pveNodes["pve-b550"].nodeName,
|
||||
provider: pveProvider,
|
||||
template: pveTemplate,
|
||||
diskDatastore: pveNodes["pve-b550"].dataStore,
|
||||
host: hostFor("pve-b550"),
|
||||
ip: master1Ip,
|
||||
},
|
||||
{
|
||||
name: "k3s-master-2",
|
||||
role: "master",
|
||||
nodeName: pveNodes["pve-esprimo"].nodeName,
|
||||
provider: pveEsprimoProvider,
|
||||
template: pveTemplate,
|
||||
diskDatastore: pveNodes["pve-esprimo"].dataStore,
|
||||
host: hostFor("pve-esprimo"),
|
||||
ip: master2Ip,
|
||||
},
|
||||
{
|
||||
name: "k3s-worker-1",
|
||||
role: "worker",
|
||||
nodeName: pveNodes["pve-b550"].nodeName,
|
||||
provider: pveProvider,
|
||||
template: pveTemplate,
|
||||
diskDatastore: pveNodes["pve-b550"].dataStore,
|
||||
host: hostFor("pve-b550"),
|
||||
ip: worker1Ip,
|
||||
longhornDiskSize: 50,
|
||||
},
|
||||
{
|
||||
name: "k3s-master-3",
|
||||
role: "master",
|
||||
nodeName: pveNodes["pve-optiplex"].nodeName,
|
||||
provider: pveBckpProvider,
|
||||
template: pveBckpTemplate,
|
||||
diskDatastore: pveNodes["pve-optiplex"].dataStore,
|
||||
host: hostFor("pve-optiplex"),
|
||||
ip: master3Ip,
|
||||
},
|
||||
{
|
||||
name: "k3s-worker-2",
|
||||
role: "worker",
|
||||
nodeName: pveNodes["pve-optiplex"].nodeName,
|
||||
provider: pveBckpProvider,
|
||||
template: pveBckpTemplate,
|
||||
diskDatastore: pveNodes["pve-optiplex"].dataStore,
|
||||
host: hostFor("pve-optiplex"),
|
||||
ip: worker2Ip,
|
||||
longhornDiskSize: 50,
|
||||
},
|
||||
{
|
||||
name: "k3s-worker-3",
|
||||
role: "worker",
|
||||
nodeName: pveNodes["pve-esprimo"].nodeName,
|
||||
provider: pveEsprimoProvider,
|
||||
template: pveEsprimoTemplate,
|
||||
diskDatastore: pveNodes["pve-esprimo"].dataStore,
|
||||
host: hostFor("pve-esprimo"),
|
||||
ip: worker3Ip,
|
||||
longhornDiskSize: 50,
|
||||
},
|
||||
@@ -292,14 +227,14 @@ const k3sVms = nodeConfigs.map(
|
||||
new proxmox.VmLegacy(
|
||||
node.name,
|
||||
{
|
||||
nodeName: node.nodeName,
|
||||
nodeName: node.host.nodeName,
|
||||
name: node.name,
|
||||
description: "k3s " + node.role + " node — managed by Pulumi",
|
||||
tags: ["k3s", node.role],
|
||||
clone: {
|
||||
vmId: node.template.vmId,
|
||||
vmId: node.host.pveTemplate.vmId,
|
||||
full: true,
|
||||
datastoreId: node.diskDatastore,
|
||||
datastoreId: node.host.dataStore,
|
||||
},
|
||||
cpu: {
|
||||
cores: 2,
|
||||
@@ -314,7 +249,7 @@ const k3sVms = nodeConfigs.map(
|
||||
disks: [
|
||||
{
|
||||
interface: "scsi0",
|
||||
datastoreId: node.diskDatastore,
|
||||
datastoreId: node.host.dataStore,
|
||||
size: 35,
|
||||
ssd: true,
|
||||
discard: "on",
|
||||
@@ -324,7 +259,7 @@ const k3sVms = nodeConfigs.map(
|
||||
: [
|
||||
{
|
||||
interface: "scsi1",
|
||||
datastoreId: node.diskDatastore,
|
||||
datastoreId: node.host.dataStore,
|
||||
size: node.longhornDiskSize,
|
||||
ssd: true,
|
||||
discard: "on",
|
||||
@@ -332,13 +267,15 @@ const k3sVms = nodeConfigs.map(
|
||||
]),
|
||||
],
|
||||
initialization: {
|
||||
datastoreId: node.diskDatastore,
|
||||
datastoreId: node.host.dataStore,
|
||||
ipConfigs: [{ ipv4: { address: "dhcp" } }],
|
||||
userAccount: {
|
||||
username: "ubuntu",
|
||||
password: k3sVmPassword,
|
||||
keys: [
|
||||
sshPvePublicKey.apply((k) => k.trim()),
|
||||
pve1SshPublicKey.apply((k) => k.trim()),
|
||||
pve2SshPublicKey.apply((k) => k.trim()),
|
||||
pve3SshPublicKey.apply((k) => k.trim()),
|
||||
ciRunnerKey.publicKeyOpenssh.apply((k) => k.trim()),
|
||||
],
|
||||
},
|
||||
@@ -347,12 +284,13 @@ const k3sVms = nodeConfigs.map(
|
||||
scsiHardware: "virtio-scsi-pci",
|
||||
serialDevices: [{}],
|
||||
vga: { type: "serial0" },
|
||||
agent: { enabled: true },
|
||||
agent: { enabled: true, timeout: "30s" },
|
||||
cdrom: { fileId: "none" },
|
||||
started: false,
|
||||
stopOnDestroy: true,
|
||||
},
|
||||
{
|
||||
provider: node.provider,
|
||||
provider: node.host.provider,
|
||||
retainOnDelete: true,
|
||||
ignoreChanges: ["clone", "started"],
|
||||
},
|
||||
|
||||
@@ -3,12 +3,10 @@ import * as command from "@pulumi/command";
|
||||
|
||||
const config = new pulumi.Config();
|
||||
|
||||
//fetch credentials from proxmox-infra
|
||||
const infraRef = new pulumi.StackReference(
|
||||
`${pulumi.getOrganization()}/proxmox-infra/dev`,
|
||||
);
|
||||
|
||||
// Proxmox API credentials — same as proxmox-infra stack
|
||||
const pve1Endpoint = infraRef.requireOutput(
|
||||
"pve1Endpoint",
|
||||
) as pulumi.Output<string>;
|
||||
@@ -255,7 +253,7 @@ const joinWorker3 = new command.remote.Command(
|
||||
const workers = [worker1Ip, worker2Ip, worker3Ip];
|
||||
const workerSshReader = [waitWorker1Ssh, waitWorker2Ssh, waitWorker3Ssh];
|
||||
|
||||
const mountLonghornDisks = workers.map((worker, i) => {
|
||||
workers.forEach((worker, i) => {
|
||||
return new command.remote.Command(
|
||||
`mount-longhorn-disk-${i}`,
|
||||
{
|
||||
@@ -287,6 +285,61 @@ const getKubeconfig = new command.remote.Command(
|
||||
{ dependsOn: [joinWorker1, joinWorker2, joinWorker3] },
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step 7: Install additional packages
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const allVms = [
|
||||
{
|
||||
id: master1VmId,
|
||||
ip: master1Ip,
|
||||
name: "master-1",
|
||||
readiness: waitMaster1Ssh,
|
||||
},
|
||||
{
|
||||
id: master2VmId,
|
||||
ip: master2Ip,
|
||||
name: "master-2",
|
||||
readiness: waitMaster2Ssh,
|
||||
},
|
||||
{
|
||||
id: master3VmId,
|
||||
ip: master3Ip,
|
||||
name: "master-3",
|
||||
readiness: waitMaster3Ssh,
|
||||
},
|
||||
{
|
||||
id: worker1VmId,
|
||||
ip: worker1Ip,
|
||||
name: "worker-1",
|
||||
readiness: waitWorker1Ssh,
|
||||
},
|
||||
{
|
||||
id: worker2VmId,
|
||||
ip: worker2Ip,
|
||||
name: "worker-2",
|
||||
readiness: waitWorker2Ssh,
|
||||
},
|
||||
{
|
||||
id: worker3VmId,
|
||||
ip: worker3Ip,
|
||||
name: "worker-3",
|
||||
readiness: waitWorker3Ssh,
|
||||
},
|
||||
];
|
||||
|
||||
allVms.forEach(({ id, ip, name, readiness }) => {
|
||||
new command.remote.Command(
|
||||
`install-packages-${name}`,
|
||||
{
|
||||
connection: conn(ip),
|
||||
create: pulumi.interpolate`sudo apt update && sudo apt install -y qemu-guest-agent && sudo systemctl enable --now qemu-guest-agent`,
|
||||
triggers: [id],
|
||||
},
|
||||
{ dependsOn: readiness },
|
||||
);
|
||||
});
|
||||
|
||||
export const kubeconfig = pulumi.secret(
|
||||
pulumi
|
||||
.all([getKubeconfig.stdout, master1Ip])
|
||||
|
||||
Reference in New Issue
Block a user