Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 37395e3a4d | |||
| c8f606fd20 | |||
| cbcb829137 | |||
| 64ef076350 | |||
| f89822e4f7 | |||
| 94be23def7 | |||
| 19af76fd2b | |||
| 354cf65657 | |||
| 61758cbb67 | |||
| c97bebd327 | |||
| 9ffc1bc94e | |||
| cc40873bb4 | |||
| c4a7ea185a | |||
| 3f874e6f97 | |||
| 5de2a16b9b | |||
| 54b97fadeb | |||
| f22cad1a37 | |||
| 113de6fb46 | |||
| 8420c33f69 | |||
| 786974fca3 |
+23
-27
@@ -2,18 +2,13 @@ name: Deploy Proxmox Infra
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- 'proxmox-infra/**'
|
|
||||||
- '.gitea/workflows/**'
|
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
paths:
|
paths:
|
||||||
- 'proxmox-infra/**'
|
- "01-proxmox-infra/**"
|
||||||
- '.gitea/workflows/**'
|
- ".gitea/workflows/**"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
preview:
|
preview:
|
||||||
@@ -27,28 +22,28 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '24'
|
node-version: "24"
|
||||||
|
|
||||||
- name: Restore Stack Config
|
- name: Restore Stack Config
|
||||||
run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > proxmox-infra/Pulumi.dev.yaml
|
run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 01-proxmox-infra/Pulumi.dev.yaml
|
||||||
|
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: npm ci
|
run: npm ci
|
||||||
working-directory: proxmox-infra
|
working-directory: 01-proxmox-infra
|
||||||
|
|
||||||
- name: Install Pulumi CLI
|
- name: Install Pulumi CLI
|
||||||
run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
|
run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
|
||||||
|
|
||||||
- name: Generate Local pfSense SDK
|
- name: Generate Local pfSense SDK
|
||||||
run: pulumi package add terraform-provider marshallford/pfsense
|
run: pulumi package add terraform-provider marshallford/pfsense 0.22.0
|
||||||
working-directory: proxmox-infra
|
working-directory: 01-proxmox-infra
|
||||||
|
|
||||||
- name: Preview
|
- name: Preview
|
||||||
uses: pulumi/actions@v5
|
uses: pulumi/actions@v5
|
||||||
with:
|
with:
|
||||||
command: preview
|
command: preview
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: proxmox-infra
|
work-dir: 01-proxmox-infra
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
@@ -56,7 +51,7 @@ jobs:
|
|||||||
deploy:
|
deploy:
|
||||||
name: Pulumi Deploy
|
name: Pulumi Deploy
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
if: github.event_name == 'workflow_dispatch'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout Code
|
- name: Checkout Code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -64,38 +59,39 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '24'
|
node-version: "24"
|
||||||
|
|
||||||
- name: Restore Stack Config
|
- name: Restore Stack Config
|
||||||
run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > proxmox-infra/Pulumi.dev.yaml
|
run: echo "${{ secrets.PROXMOX_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 01-proxmox-infra/Pulumi.dev.yaml
|
||||||
|
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: npm ci
|
run: npm ci
|
||||||
working-directory: proxmox-infra
|
working-directory: 01-proxmox-infra
|
||||||
|
|
||||||
- name: Install Pulumi CLI
|
- name: Install Pulumi CLI
|
||||||
run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
|
run: curl -fsSL https://get.pulumi.com | sh && echo "$HOME/.pulumi/bin" >> $GITHUB_PATH
|
||||||
|
|
||||||
- name: Generate Local pfSense SDK
|
- name: Generate Local pfSense SDK
|
||||||
run: pulumi package add terraform-provider marshallford/pfsense
|
run: pulumi package add terraform-provider marshallford/pfsense 0.22.0
|
||||||
working-directory: proxmox-infra
|
working-directory: 01-proxmox-infra
|
||||||
|
|
||||||
- name: Refresh State
|
- name: Refresh State
|
||||||
uses: pulumi/actions@v5
|
run: |
|
||||||
with:
|
pulumi login "$PULUMI_BACKEND_URL"
|
||||||
command: refresh
|
pulumi refresh --yes --stack dev \
|
||||||
stack-name: dev
|
--target 'urn:pulumi:dev::proxmox-infra::proxmoxve*' \
|
||||||
work-dir: proxmox-infra
|
--target 'urn:pulumi:dev::proxmox-infra::tls*'
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
working-directory: 01-proxmox-infra
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
|
PULUMI_BACKEND_URL: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
|
|
||||||
- name: Deploy
|
- name: Deploy
|
||||||
uses: pulumi/actions@v5
|
uses: pulumi/actions@v5
|
||||||
with:
|
with:
|
||||||
command: up
|
command: up
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: proxmox-infra
|
work-dir: 01-proxmox-infra
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
+10
-16
@@ -2,18 +2,12 @@ name: Deploy k8s Bootstrap
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- "k8s-bootstrap/**"
|
|
||||||
- ".gitea/workflows/deploy-k8s-bootstrap.yaml"
|
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
paths:
|
paths:
|
||||||
- "k8s-bootstrap/**"
|
- "02-k8s-bootstrap/**"
|
||||||
- ".gitea/workflows/deploy-k8s-bootstrap.yaml"
|
- ".gitea/workflows/02-deploy-k8s-bootstrap.yaml"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
preview:
|
preview:
|
||||||
@@ -30,18 +24,18 @@ jobs:
|
|||||||
node-version: "24"
|
node-version: "24"
|
||||||
|
|
||||||
- name: Restore Stack Config
|
- name: Restore Stack Config
|
||||||
run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
|
run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > 02-k8s-bootstrap/Pulumi.dev.yaml
|
||||||
|
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: npm install
|
run: npm install
|
||||||
working-directory: k8s-bootstrap
|
working-directory: 02-k8s-bootstrap
|
||||||
|
|
||||||
- name: Preview
|
- name: Preview
|
||||||
uses: pulumi/actions@v5
|
uses: pulumi/actions@v5
|
||||||
with:
|
with:
|
||||||
command: preview
|
command: preview
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: k8s-bootstrap
|
work-dir: 02-k8s-bootstrap
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
@@ -49,7 +43,7 @@ jobs:
|
|||||||
deploy:
|
deploy:
|
||||||
name: Bootstrap k3s Cluster
|
name: Bootstrap k3s Cluster
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
if: github.event_name == 'workflow_dispatch'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout Code
|
- name: Checkout Code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -60,18 +54,18 @@ jobs:
|
|||||||
node-version: "24"
|
node-version: "24"
|
||||||
|
|
||||||
- name: Restore Stack Config
|
- name: Restore Stack Config
|
||||||
run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > k8s-bootstrap/Pulumi.dev.yaml
|
run: echo "${{ secrets.K8S_BOOTSTRAP_PULUMI_DEV_YAML }}" | base64 -d > 02-k8s-bootstrap/Pulumi.dev.yaml
|
||||||
|
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: npm install
|
run: npm install
|
||||||
working-directory: k8s-bootstrap
|
working-directory: 02-k8s-bootstrap
|
||||||
|
|
||||||
- name: Refresh State
|
- name: Refresh State
|
||||||
uses: pulumi/actions@v5
|
uses: pulumi/actions@v5
|
||||||
with:
|
with:
|
||||||
command: refresh
|
command: refresh
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: k8s-bootstrap
|
work-dir: 02-k8s-bootstrap
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
@@ -81,7 +75,7 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
command: up
|
command: up
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: k8s-bootstrap
|
work-dir: 02-k8s-bootstrap
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
@@ -6,13 +6,13 @@ on:
|
|||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
paths:
|
paths:
|
||||||
- 'k8s-infra/**'
|
- '03-k8s-infra/**'
|
||||||
- '.gitea/workflows/**'
|
- '.gitea/workflows/**'
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
paths:
|
paths:
|
||||||
- 'k8s-infra/**'
|
- '03-k8s-infra/**'
|
||||||
- '.gitea/workflows/**'
|
- '.gitea/workflows/**'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
@@ -30,21 +30,21 @@ jobs:
|
|||||||
node-version: '24'
|
node-version: '24'
|
||||||
|
|
||||||
- name: Restore Stack Config
|
- name: Restore Stack Config
|
||||||
run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > k8s-infra/Pulumi.dev.yaml
|
run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 03-k8s-infra/Pulumi.dev.yaml
|
||||||
|
|
||||||
- name: Install Helm
|
- name: Install Helm
|
||||||
uses: azure/setup-helm@v4
|
uses: azure/setup-helm@v4
|
||||||
|
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: npm ci
|
run: npm ci
|
||||||
working-directory: k8s-infra
|
working-directory: 03-k8s-infra
|
||||||
|
|
||||||
- name: Preview
|
- name: Preview
|
||||||
uses: pulumi/actions@v5
|
uses: pulumi/actions@v5
|
||||||
with:
|
with:
|
||||||
command: preview
|
command: preview
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: k8s-infra
|
work-dir: 03-k8s-infra
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
@@ -63,21 +63,21 @@ jobs:
|
|||||||
node-version: '24'
|
node-version: '24'
|
||||||
|
|
||||||
- name: Restore Stack Config
|
- name: Restore Stack Config
|
||||||
run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > k8s-infra/Pulumi.dev.yaml
|
run: echo "${{ secrets.K8S_INFRA_PULUMI_DEV_YAML }}" | base64 -d > 03-k8s-infra/Pulumi.dev.yaml
|
||||||
|
|
||||||
- name: Install Helm
|
- name: Install Helm
|
||||||
uses: azure/setup-helm@v4
|
uses: azure/setup-helm@v4
|
||||||
|
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: npm ci
|
run: npm ci
|
||||||
working-directory: k8s-infra
|
working-directory: 03-k8s-infra
|
||||||
|
|
||||||
- name: Refresh State
|
- name: Refresh State
|
||||||
uses: pulumi/actions@v5
|
uses: pulumi/actions@v5
|
||||||
with:
|
with:
|
||||||
command: refresh
|
command: refresh
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: k8s-infra
|
work-dir: 03-k8s-infra
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
@@ -87,7 +87,7 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
command: up
|
command: up
|
||||||
stack-name: dev
|
stack-name: dev
|
||||||
work-dir: k8s-infra
|
work-dir: 03-k8s-infra
|
||||||
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
cloud-url: ${{ secrets.PULUMI_BACKEND_URL }}
|
||||||
env:
|
env:
|
||||||
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
|
||||||
@@ -71,7 +71,7 @@ const ubuntuImagePve = new proxmox.download.File(
|
|||||||
contentType: "import",
|
contentType: "import",
|
||||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
fileName: "noble-server-cloudimg-amd64.qcow2",
|
||||||
url: ubuntuNobleUrl,
|
url: ubuntuNobleUrl,
|
||||||
overwrite: true,
|
overwrite: false,
|
||||||
overwriteUnmanaged: true,
|
overwriteUnmanaged: true,
|
||||||
},
|
},
|
||||||
{ provider: pveProvider },
|
{ provider: pveProvider },
|
||||||
@@ -85,7 +85,7 @@ const ubuntuImagePveBckp = new proxmox.download.File(
|
|||||||
contentType: "import",
|
contentType: "import",
|
||||||
fileName: "noble-server-cloudimg-amd64.qcow2",
|
fileName: "noble-server-cloudimg-amd64.qcow2",
|
||||||
url: ubuntuNobleUrl,
|
url: ubuntuNobleUrl,
|
||||||
overwrite: true,
|
overwrite: false,
|
||||||
overwriteUnmanaged: true,
|
overwriteUnmanaged: true,
|
||||||
},
|
},
|
||||||
{ provider: pveBckpProvider },
|
{ provider: pveBckpProvider },
|
||||||
@@ -135,7 +135,7 @@ const pveTemplate = new proxmox.VmLegacy(
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{ provider: pveProvider },
|
{ provider: pveProvider, ignoreChanges: ["disks"] },
|
||||||
);
|
);
|
||||||
|
|
||||||
const pveBckpTemplate = new proxmox.VmLegacy(
|
const pveBckpTemplate = new proxmox.VmLegacy(
|
||||||
@@ -155,7 +155,7 @@ const pveBckpTemplate = new proxmox.VmLegacy(
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{ provider: pveBckpProvider },
|
{ provider: pveBckpProvider, ignoreChanges: ["disks"] },
|
||||||
);
|
);
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -279,7 +279,7 @@ const k3sVms = nodeConfigs.map(
|
|||||||
{
|
{
|
||||||
provider: node.provider,
|
provider: node.provider,
|
||||||
retainOnDelete: true,
|
retainOnDelete: true,
|
||||||
ignoreChanges: ["clone"],
|
ignoreChanges: ["clone", "started", "disks"],
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
@@ -67,8 +67,9 @@ const certManager = new k8s.helm.v3.Release(
|
|||||||
repositoryOpts: { repo: "https://charts.jetstack.io" },
|
repositoryOpts: { repo: "https://charts.jetstack.io" },
|
||||||
namespace: "cert-manager",
|
namespace: "cert-manager",
|
||||||
createNamespace: true,
|
createNamespace: true,
|
||||||
|
timeout: 600,
|
||||||
values: {
|
values: {
|
||||||
installCRDs: true,
|
crds: { enabled: true },
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
opts(),
|
opts(),
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Homelab Infrastructure as Code
|
# Homelab Infrastructure as Code
|
||||||
|
|
||||||
A Pulumi-based IaC template for managing a Proxmox homelab. The goal is to replace manual GUI configuration and ad-hoc YAML stacks (LXC, VM, Docker, etc.) with version-controlled, reproducible infrastructure — starting with a highly available k3s cluster across multiple Proxmox nodes.
|
A Pulumi-based IaC template for managing a Proxmox homelab. The goal is to replace manual GUI configuration and ad-hoc YAML stacks with version-controlled, reproducible infrastructure — a highly available k3s cluster across multiple Proxmox nodes, bootstrapped and configured end-to-end from code.
|
||||||
|
|
||||||
This repo is intentionally abstract: credentials are never hardcoded, making it easy to fork and adapt as a template for your own homelab.
|
This repo is intentionally abstract: credentials are never hardcoded, making it easy to fork and adapt as a template for your own homelab.
|
||||||
|
|
||||||
@@ -16,20 +16,30 @@ This repo is intentionally abstract: credentials are never hardcoded, making it
|
|||||||
|
|
||||||
```
|
```
|
||||||
.
|
.
|
||||||
├── proxmox-infra/ # Pulumi TypeScript stack — VMs & LXC on Proxmox
|
├── proxmox-infra/ # Stack 1 — VMs & DHCP on Proxmox + pfSense
|
||||||
│ ├── index.ts # All Pulumi resources
|
│ ├── index.ts
|
||||||
│ ├── Pulumi.yaml # Stack project definition
|
│ ├── Pulumi.yaml
|
||||||
│ ├── Pulumi.dev.yaml # Encrypted stack config (gitignored)
|
│ └── sdks/pfsense/ # Locally bundled @pulumi/pfsense SDK
|
||||||
│ └── sdks/
|
├── k8s-bootstrap/ # Stack 2 — Install k3s on the VMs over SSH
|
||||||
│ └── pfsense/ # Locally bundled @pulumi/pfsense SDK
|
│ ├── index.ts
|
||||||
├── .gitea/
|
│ └── Pulumi.yaml
|
||||||
│ └── workflows/
|
├── k8s-infra/ # Stack 3 — Cluster-level infrastructure via Helm
|
||||||
│ └── deploy-proxmox-infra.yaml # Gitea Actions CI/CD pipeline
|
│ ├── index.ts
|
||||||
|
│ └── Pulumi.yaml
|
||||||
|
└── .gitea/workflows/ # Gitea Actions — one workflow per stack
|
||||||
```
|
```
|
||||||
|
|
||||||
## Current stack: `proxmox-infra`
|
## Stack overview
|
||||||
|
|
||||||
Provisions a 5-node k3s cluster spread across two Proxmox hosts (`pve` and `pve-bckp`, third bare metal host to be added later for actual parity):
|
The three stacks are deployed in order and linked via Pulumi StackReferences — outputs from one stack flow automatically into the next.
|
||||||
|
|
||||||
|
```
|
||||||
|
proxmox-infra → k8s-bootstrap → k8s-infra
|
||||||
|
```
|
||||||
|
|
||||||
|
### Stack 1: `proxmox-infra`
|
||||||
|
|
||||||
|
Provisions a 5-node k3s cluster spread across two Proxmox hosts:
|
||||||
|
|
||||||
| VM name | Role | Proxmox node |
|
| VM name | Role | Proxmox node |
|
||||||
| ------------ | ------ | ------------ |
|
| ------------ | ------ | ------------ |
|
||||||
@@ -39,43 +49,64 @@ Provisions a 5-node k3s cluster spread across two Proxmox hosts (`pve` and `pve-
|
|||||||
| k3s-master-3 | master | pve-bckp |
|
| k3s-master-3 | master | pve-bckp |
|
||||||
| k3s-worker-2 | worker | pve-bckp |
|
| k3s-worker-2 | worker | pve-bckp |
|
||||||
|
|
||||||
Each node is a full clone of an Ubuntu Noble (24.04) cloud-image template, with cloud-init injecting hostname, user credentials, and SSH keys at boot. Each VM's MAC address is registered as a DHCPv4 static mapping in pfSense so that nodes always receive their designated IPs.
|
Each VM is a full clone of an Ubuntu Noble (24.04) cloud-image template. Cloud-init injects the hostname, user credentials, and SSH keys at boot. An ED25519 SSH keypair is generated once and stored in Pulumi state — the public key goes into every VM, the private key is exported as a stack output for `k8s-bootstrap` to consume via StackReference.
|
||||||
|
|
||||||
An ED25519 SSH key pair is generated once and stored in Pulumi state. The public key is injected into every VM at boot; the private key is exported as a stack output so `k8s-bootstrap` can consume it via StackReference without any manual key distribution.
|
Each VM's MAC address is registered as a DHCPv4 static mapping in pfSense so nodes always receive their designated IPs.
|
||||||
|
|
||||||
**Tech stack:**
|
**Providers:** `@muhlba91/pulumi-proxmoxve` v8.x, `@pulumi/pfsense` (locally bundled), `@pulumi/tls`
|
||||||
|
|
||||||
- [Pulumi](https://www.pulumi.com/) with TypeScript
|
**Exports:** `vmIds`, `ciRunnerPrivateKey`, `pve1Endpoint`, `pve1ApiToken`, `pve2Endpoint`, `pve2ApiToken`, `master1Ip` … `worker2Ip`
|
||||||
- [`@muhlba91/pulumi-proxmoxve`](https://github.com/muhlba91/pulumi-provider-proxmoxve) v8.x community provider
|
|
||||||
- [`@pulumi/pfsense`](https://github.com/marshallford/terraform-provider-pfsense) — locally bundled SDK bridged from the Terraform pfSense provider; installed automatically via `npm install`
|
### Stack 2: `k8s-bootstrap`
|
||||||
- [`@pulumi/tls`](https://www.pulumi.com/registry/packages/tls/) — SSH key pair generation
|
|
||||||
- Self-hosted Pulumi state backend (PostgreSQL)
|
Installs k3s on the Proxmox VMs over SSH using `@pulumi/command`. Reads VM IDs, Proxmox credentials, and the SSH private key from `proxmox-infra` via StackReference — no manual key distribution needed.
|
||||||
- Gitea Actions for CI/CD
|
|
||||||
|
Deployment sequence:
|
||||||
|
|
||||||
|
1. Start all 5 VMs via the Proxmox REST API
|
||||||
|
2. Wait for port 22 to open on each node
|
||||||
|
3. Install k3s on `k3s-master-1` with `--cluster-init` (embedded etcd)
|
||||||
|
4. Join `k3s-master-2` and `k3s-master-3` as additional etcd nodes
|
||||||
|
5. Join `k3s-worker-1` and `k3s-worker-2` as agent nodes
|
||||||
|
6. Read `/etc/rancher/k3s/k3s.yaml` from master-1, patch the server URL, export as a secret stack output
|
||||||
|
|
||||||
|
**Exports:** `kubeconfig` (secret)
|
||||||
|
|
||||||
|
### Stack 3: `k8s-infra`
|
||||||
|
|
||||||
|
Deploys cluster-level infrastructure via Helm and the `@pulumi/kubernetes` provider. Reads `kubeconfig` from `k8s-bootstrap` via StackReference.
|
||||||
|
|
||||||
|
| Component | What it does |
|
||||||
|
| -------------------- | -------------------------------------------------------------------------- |
|
||||||
|
| NFS CSI Driver | `csi-driver-nfs` Helm chart (v4.12.0) — enables dynamic NFS-backed PVCs |
|
||||||
|
| TrueNAS StorageClass | `truenas-nfs` StorageClass backed by an NFS share on TrueNAS |
|
||||||
|
| cert-manager | `cert-manager` Helm chart — certificate lifecycle management |
|
||||||
|
| ClusterIssuer | `letsencrypt-prod` — DNS-01 via Cloudflare, issues Let's Encrypt TLS certs |
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
- [Pulumi CLI](https://www.pulumi.com/docs/install/) installed
|
- [Pulumi CLI](https://www.pulumi.com/docs/install/) installed
|
||||||
- Node.js 18+ and npm
|
- Node.js 18+ and npm
|
||||||
- Access to a Proxmox node with an API token
|
- Two Proxmox nodes with API tokens
|
||||||
- pfSense instance with API credentials (used for DHCPv4 static mappings)
|
- pfSense with REST API credentials (used for DHCPv4 static mapping)
|
||||||
|
- TrueNAS with an NFS share (no API key needed — CSI driver connects via NFS protocol directly)
|
||||||
|
- A Cloudflare account with an API token scoped to DNS edit on your zone
|
||||||
- A self-hosted Pulumi state backend (PostgreSQL connection string)
|
- A self-hosted Pulumi state backend (PostgreSQL connection string)
|
||||||
- Gitea instance for CI/CD (optional for local use)
|
- Gitea instance for CI/CD (optional for local use)
|
||||||
|
|
||||||
## Getting started
|
## Getting started
|
||||||
|
|
||||||
### 1. Clone and install
|
Deploy the stacks **in order**. Each stack must be fully deployed before the next one runs.
|
||||||
|
|
||||||
|
### Stack 1 — `proxmox-infra`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone <your-repo-url>
|
|
||||||
cd proxmox-infra
|
cd proxmox-infra
|
||||||
npm install
|
npm install # also compiles the bundled pfSense SDK
|
||||||
|
pulumi stack init dev
|
||||||
```
|
```
|
||||||
|
|
||||||
> **pfSense SDK** — The `@pulumi/pfsense` SDK is bundled locally under `sdks/pfsense/` and referenced as a `file:` dependency in `package.json`. Running `npm install` compiles it automatically via its postinstall hook. No separate installation or build step is required.
|
Set secrets:
|
||||||
|
|
||||||
### 2. Configure credentials
|
|
||||||
|
|
||||||
All secrets are stored as encrypted Pulumi config values — never in plain environment variables or committed files.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Proxmox API credentials
|
# Proxmox API credentials
|
||||||
@@ -101,31 +132,68 @@ pulumi config set --secret master3Ip <ip-for-k3s-master-3>
|
|||||||
pulumi config set --secret worker2Ip <ip-for-k3s-worker-2>
|
pulumi config set --secret worker2Ip <ip-for-k3s-worker-2>
|
||||||
```
|
```
|
||||||
|
|
||||||
Pulumi encrypts these values into `Pulumi.dev.yaml` using your `PULUMI_CONFIG_PASSPHRASE`.
|
Deploy:
|
||||||
|
|
||||||
### 3. Set the state backend
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
export PULUMI_BACKEND_URL=postgresql://<user>:<pass>@<host>/<db>
|
export PULUMI_BACKEND_URL=postgresql://<user>:<pass>@<host>/<db>
|
||||||
export PULUMI_CONFIG_PASSPHRASE=<your-passphrase>
|
export PULUMI_CONFIG_PASSPHRASE=<your-passphrase>
|
||||||
|
|
||||||
|
pulumi preview # inspect before touching anything
|
||||||
|
pulumi up --yes
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. Preview and deploy
|
### Stack 2 — `k8s-bootstrap`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# See what will change before touching anything
|
cd ../k8s-bootstrap
|
||||||
pulumi preview
|
npm install
|
||||||
|
pulumi stack init dev
|
||||||
|
|
||||||
# Sync Pulumi state with actual Proxmox state (run after any manual GUI changes)
|
pulumi config set --secret k3sToken "$(openssl rand -hex 32)"
|
||||||
pulumi refresh --yes
|
```
|
||||||
|
|
||||||
# Deploy
|
Node IPs and Proxmox credentials are read automatically from `proxmox-infra` outputs via StackReference — do not set them here.
|
||||||
pulumi refresh --yes && pulumi up --yes
|
|
||||||
|
```bash
|
||||||
|
pulumi up --yes
|
||||||
|
```
|
||||||
|
|
||||||
|
### Stack 3 — `k8s-infra`
|
||||||
|
|
||||||
|
**TrueNAS one-time setup** (before deploying this stack):
|
||||||
|
|
||||||
|
1. Create dataset `tank/k8s`
|
||||||
|
2. Add an NFS share for that dataset
|
||||||
|
|
||||||
|
- Acl Type: Set this to POSIX (or generic Unix permissions) rather than SMB/NFSv4 ACLs if you are on TrueNAS SCALE, as Kubernetes handles basic Unix permissions natively.
|
||||||
|
- Maproot User / Maproot Group: Set this to root and root
|
||||||
|
- Allowed Hosts/Networks: Restrict this share specifically to the IP addresses or the CIDR block of your Proxmox K3s nodes (e.g., 192.168.1.50)
|
||||||
|
|
||||||
|
3. In Network → Allowed Networks, permit your LAN subnet (e.g. `192.168.1.0/24`)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ../k8s-infra
|
||||||
|
npm install
|
||||||
|
pulumi stack init dev
|
||||||
|
|
||||||
|
# kubeconfig from k8s-bootstrap
|
||||||
|
KUBECONFIG=$(cd ../k8s-bootstrap && pulumi stack output kubeconfig --show-secrets)
|
||||||
|
pulumi config set --secret kubeconfig "$KUBECONFIG"
|
||||||
|
|
||||||
|
# TrueNAS NFS
|
||||||
|
pulumi config set --secret truenasHost <truenas-ip>
|
||||||
|
pulumi config set --secret truenasNfsPath /mnt/tank/k8s
|
||||||
|
|
||||||
|
# cert-manager + Cloudflare DNS-01
|
||||||
|
pulumi config set --secret cloudflareApiToken <cf-token>
|
||||||
|
pulumi config set --secret letsencryptEmail <your-email>
|
||||||
|
|
||||||
|
pulumi up --yes
|
||||||
```
|
```
|
||||||
|
|
||||||
## CI/CD (Gitea Actions)
|
## CI/CD (Gitea Actions)
|
||||||
|
|
||||||
The workflow at `.gitea/workflows/deploy-proxmox-infra.yaml` runs automatically:
|
One workflow per stack under `.gitea/workflows/`. Each triggers on changes to its own stack directory.
|
||||||
|
|
||||||
| Event | Action |
|
| Event | Action |
|
||||||
| --------------------- | ------------------------------ |
|
| --------------------- | ------------------------------ |
|
||||||
@@ -135,34 +203,37 @@ The workflow at `.gitea/workflows/deploy-proxmox-infra.yaml` runs automatically:
|
|||||||
|
|
||||||
### Required Gitea secrets
|
### Required Gitea secrets
|
||||||
|
|
||||||
Configure these under **Settings → Actions → Secrets** in your Gitea repo:
|
Configure under **Settings → Actions → Secrets**:
|
||||||
|
|
||||||
| Secret | Description |
|
| Secret | Used by | Description |
|
||||||
| -------------------------- | -------------------------------------------------- |
|
| ------------------------------- | --------------------------- | ---------------------------------------------- |
|
||||||
| `PULUMI_BACKEND_URL` | PostgreSQL connection string for the state backend |
|
| `PULUMI_BACKEND_URL` | all workflows | PostgreSQL connection string for state backend |
|
||||||
| `PULUMI_CONFIG_PASSPHRASE` | Passphrase to decrypt secrets in `Pulumi.dev.yaml` |
|
| `PULUMI_CONFIG_PASSPHRASE` | all workflows | Passphrase to decrypt secrets |
|
||||||
| `PULUMI_DEV_YAML` | Base64-encoded content of `Pulumi.dev.yaml` |
|
| `PROXMOX_INFRA_PULUMI_DEV_YAML` | `deploy-proxmox-infra.yaml` | Base64-encoded `proxmox-infra/Pulumi.dev.yaml` |
|
||||||
|
| `K8S_BOOTSTRAP_PULUMI_DEV_YAML` | `deploy-k8s-bootstrap.yaml` | Base64-encoded `k8s-bootstrap/Pulumi.dev.yaml` |
|
||||||
|
| `K8S_INFRA_PULUMI_DEV_YAML` | `deploy-k8s-infra.yaml` | Base64-encoded `k8s-infra/Pulumi.dev.yaml` |
|
||||||
|
|
||||||
`Pulumi.dev.yaml` is gitignored because it contains your encryption salt. Whenever it changes (e.g. after adding or rotating a secret), re-encode it and paste the output into the Gitea secret:
|
`Pulumi.dev.yaml` files are gitignored (they contain your encryption salt). Whenever one changes, re-encode and paste into the corresponding Gitea secret:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
base64 -w 0 proxmox-infra/Pulumi.dev.yaml
|
base64 -w 0 proxmox-infra/Pulumi.dev.yaml
|
||||||
|
base64 -w 0 k8s-bootstrap/Pulumi.dev.yaml
|
||||||
|
base64 -w 0 k8s-infra/Pulumi.dev.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
## Adapting this as a template
|
## Adapting this as a template
|
||||||
|
|
||||||
1. Fork or copy the repo
|
1. Fork or copy the repo
|
||||||
2. Update node names (`pve`, `pve-bckp`) and datastore IDs in `index.ts` to match your setup
|
2. Update Proxmox node names (`pve`, `pve-bckp`) and datastore IDs in `proxmox-infra/index.ts`
|
||||||
3. Add or remove VMs from the `nodeConfigs` array
|
3. Add or remove VMs from the `nodeConfigs` array
|
||||||
4. Set your own secrets with `pulumi config set --secret`
|
4. Set your own secrets with `pulumi config set --secret`
|
||||||
5. Point the CI/CD workflow at your own Git instance
|
5. Point the CI/CD workflows at your own Gitea instance
|
||||||
|
|
||||||
## Roadmap
|
## Roadmap
|
||||||
|
|
||||||
- LXC container management
|
- LXC container management
|
||||||
|
- `k8s-apps` stack — application deployments on the cluster
|
||||||
- Docker / Compose stack provisioning
|
- Docker / Compose stack provisioning
|
||||||
- Firewall rules (pfSense)
|
|
||||||
- Automated k3s bootstrapping (kubeconfig export)
|
|
||||||
- Additional worker nodes and storage volumes
|
|
||||||
- Migrate secrets management to [OpenBao](https://openbao.org/) — replace `PULUMI_CONFIG_PASSPHRASE` and manual `Pulumi.dev.yaml` encoding with a self-hosted vault
|
- Migrate secrets management to [OpenBao](https://openbao.org/) — replace `PULUMI_CONFIG_PASSPHRASE` and manual `Pulumi.dev.yaml` encoding with a self-hosted vault
|
||||||
- Add a third bare metal proxmox instance to create an actual 3 node parity.
|
- Add a third bare metal Proxmox node for true 3-node HA parity
|
||||||
|
|||||||
Reference in New Issue
Block a user