409 lines
11 KiB
TypeScript
409 lines
11 KiB
TypeScript
import * as pulumi from "@pulumi/pulumi";
|
|
import * as k8s from "@pulumi/kubernetes";
|
|
|
|
const config = new pulumi.Config();
|
|
|
|
const bootstrapRef = new pulumi.StackReference(
|
|
`${pulumi.getOrganization()}/k8s-bootstrap/dev`,
|
|
);
|
|
|
|
const kubeconfig = bootstrapRef.requireOutput("kubeconfig");
|
|
|
|
export const nfsStorageClass = "truenas-nfs";
|
|
const truenasHost = config.requireSecret("truenasHost");
|
|
const truenasNfsPath = config.requireSecret("truenasNfsPath");
|
|
|
|
const cloudflareToken = config.requireSecret("cloudflareApiToken");
|
|
const letsencryptEmail = config.requireSecret("letsencryptEmail");
|
|
|
|
const kubeVipAddress = config.requireSecret("kubeVipAddress");
|
|
const kubeVipInterface = config.require("kubeVipInterface");
|
|
|
|
const k8sProvider = new k8s.Provider("k3s", { kubeconfig });
|
|
const opts = (extras?: pulumi.ResourceOptions): pulumi.ResourceOptions => ({
|
|
provider: k8sProvider,
|
|
...extras,
|
|
});
|
|
|
|
// ── NFS CSI Driver ────────────────────────────────────────────────────────
|
|
|
|
const nfsCsiDriver = new k8s.helm.v3.Release(
|
|
"nfs-csi-driver",
|
|
{
|
|
name: "csi-driver-nfs",
|
|
chart: "csi-driver-nfs",
|
|
repositoryOpts: {
|
|
repo: "https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts",
|
|
},
|
|
namespace: "kube-system",
|
|
version: "4.12.0",
|
|
values: {
|
|
kubeletDir: "/var/lib/kubelet",
|
|
},
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
new k8s.storage.v1.StorageClass(
|
|
nfsStorageClass,
|
|
{
|
|
metadata: { name: nfsStorageClass },
|
|
provisioner: "nfs.csi.k8s.io",
|
|
parameters: {
|
|
server: truenasHost,
|
|
share: truenasNfsPath,
|
|
mountPermissions: "0",
|
|
},
|
|
reclaimPolicy: "Retain",
|
|
volumeBindingMode: "Immediate",
|
|
allowVolumeExpansion: true,
|
|
mountOptions: ["nfsvers=4.1"],
|
|
},
|
|
opts({ dependsOn: [nfsCsiDriver] }),
|
|
);
|
|
|
|
// ── cert-manager ──────────────────────────────────────────────────────────
|
|
|
|
const certManager = new k8s.helm.v3.Release(
|
|
"cert-manager",
|
|
{
|
|
name: "cert-manager",
|
|
chart: "cert-manager",
|
|
repositoryOpts: { repo: "https://charts.jetstack.io" },
|
|
namespace: "cert-manager",
|
|
createNamespace: true,
|
|
timeout: 600,
|
|
values: {
|
|
crds: { enabled: true },
|
|
},
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
// ── Cloudflare token secret + ClusterIssuer ───────────────────────────────
|
|
|
|
const cfSecret = new k8s.core.v1.Secret(
|
|
"cloudflare-token",
|
|
{
|
|
metadata: { name: "cloudflare-api-token", namespace: "cert-manager" },
|
|
stringData: { "api-token": cloudflareToken },
|
|
},
|
|
opts({ dependsOn: [certManager] }),
|
|
);
|
|
|
|
new k8s.apiextensions.CustomResource(
|
|
"letsencrypt-prod",
|
|
{
|
|
apiVersion: "cert-manager.io/v1",
|
|
kind: "ClusterIssuer",
|
|
metadata: { name: "letsencrypt-prod" },
|
|
spec: {
|
|
acme: {
|
|
server: "https://acme-v02.api.letsencrypt.org/directory",
|
|
email: letsencryptEmail,
|
|
privateKeySecretRef: { name: "letsencrypt-prod-account-key" },
|
|
solvers: [
|
|
{
|
|
dns01: {
|
|
cloudflare: {
|
|
apiTokenSecretRef: {
|
|
name: cfSecret.metadata.name,
|
|
key: "api-token",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
opts({ dependsOn: [certManager, cfSecret] }),
|
|
);
|
|
|
|
// ── Longhorn ──────────────────────────────────────────────────────────────
|
|
|
|
const longhornRelease = new k8s.helm.v3.Release(
|
|
"longhorn",
|
|
{
|
|
name: "longhorn",
|
|
chart: "longhorn",
|
|
repositoryOpts: { repo: "https://charts.longhorn.io" },
|
|
namespace: "longhorn-system",
|
|
createNamespace: true,
|
|
version: "1.12.0",
|
|
values: {
|
|
defaultSettings: {
|
|
defaultReplicaCount: 3,
|
|
},
|
|
},
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
new k8s.apiextensions.CustomResource(
|
|
"longhorn-pod-deletion-policy",
|
|
{
|
|
apiVersion: "longhorn.io/v1beta2",
|
|
kind: "Setting",
|
|
metadata: {
|
|
name: "pod-deletion-policy-when-node-is-down",
|
|
namespace: "longhorn-system",
|
|
},
|
|
value: "delete-pod-and-force-delete-pod",
|
|
},
|
|
{
|
|
provider: k8sProvider,
|
|
dependsOn: [longhornRelease],
|
|
},
|
|
);
|
|
|
|
// ── kube-vip ────────────
|
|
|
|
const kubeVipNamespace = "kube-system";
|
|
const controlPlaneTolerations = [
|
|
{ key: "node-role.kubernetes.io/master", effect: "NoSchedule" },
|
|
{ key: "node-role.kubernetes.io/control-plane", effect: "NoSchedule" },
|
|
];
|
|
|
|
const kubeVipServiceAccount = new k8s.core.v1.ServiceAccount(
|
|
"kube-vip",
|
|
{ metadata: { name: "kube-vip", namespace: kubeVipNamespace } },
|
|
opts(),
|
|
);
|
|
|
|
const kubeVipClusterRole = new k8s.rbac.v1.ClusterRole(
|
|
"kube-vip-role",
|
|
{
|
|
metadata: {
|
|
name: "system:kube-vip-role",
|
|
annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" },
|
|
},
|
|
rules: [
|
|
{ apiGroups: [""], resources: ["services/status"], verbs: ["update"] },
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["services", "endpoints"],
|
|
verbs: ["list", "get", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["list", "get", "watch", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["coordination.k8s.io"],
|
|
resources: ["leases"],
|
|
verbs: ["list", "get", "watch", "update", "create"],
|
|
},
|
|
{
|
|
apiGroups: ["discovery.k8s.io"],
|
|
resources: ["endpointslices"],
|
|
verbs: ["list", "get", "watch", "update"],
|
|
},
|
|
{ apiGroups: [""], resources: ["pods"], verbs: ["list"] },
|
|
],
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
const kubeVipClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding(
|
|
"kube-vip-binding",
|
|
{
|
|
metadata: { name: "system:kube-vip-binding" },
|
|
roleRef: {
|
|
apiGroup: "rbac.authorization.k8s.io",
|
|
kind: "ClusterRole",
|
|
name: kubeVipClusterRole.metadata.name,
|
|
},
|
|
subjects: [
|
|
{
|
|
kind: "ServiceAccount",
|
|
name: kubeVipServiceAccount.metadata.name,
|
|
namespace: kubeVipNamespace,
|
|
},
|
|
],
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
const kubeVipLabels = { name: "kube-vip-ds" };
|
|
|
|
const kubeVip = new k8s.apps.v1.DaemonSet(
|
|
"kube-vip",
|
|
{
|
|
metadata: { name: "kube-vip-ds", namespace: kubeVipNamespace },
|
|
spec: {
|
|
selector: { matchLabels: kubeVipLabels },
|
|
template: {
|
|
metadata: { labels: kubeVipLabels },
|
|
spec: {
|
|
serviceAccountName: kubeVipServiceAccount.metadata.name,
|
|
hostNetwork: true,
|
|
tolerations: [
|
|
...controlPlaneTolerations.map((t) => ({
|
|
...t,
|
|
operator: "Exists",
|
|
})),
|
|
{ effect: "NoExecute", operator: "Exists" },
|
|
],
|
|
containers: [
|
|
{
|
|
name: "kube-vip",
|
|
image: "ghcr.io/kube-vip/kube-vip:v1.2.0",
|
|
imagePullPolicy: "Always",
|
|
args: ["manager"],
|
|
env: [
|
|
{ name: "vip_arp", value: "true" },
|
|
{ name: "vip_interface", value: kubeVipInterface },
|
|
{ name: "vip_subnet", value: "32" },
|
|
{ name: "cp_enable", value: "false" },
|
|
{ name: "svc_enable", value: "true" },
|
|
{ name: "svc_election", value: "true" },
|
|
{ name: "vip_leaderelection", value: "true" },
|
|
],
|
|
securityContext: {
|
|
capabilities: { add: ["NET_ADMIN", "NET_RAW"] },
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
opts({ dependsOn: [kubeVipClusterRoleBinding] }),
|
|
);
|
|
|
|
// -- kube-vip-cloud-provider: allocates LoadBalancer IPs for Services on bare-metal
|
|
// k3s (replacing the disabled ServiceLB). The DaemonSet above only *announces* IPs
|
|
// via ARP; this controller is what actually assigns them to Service.status.
|
|
|
|
const cloudProviderServiceAccount = new k8s.core.v1.ServiceAccount(
|
|
"kube-vip-cloud-controller",
|
|
{
|
|
metadata: {
|
|
name: "kube-vip-cloud-controller",
|
|
namespace: kubeVipNamespace,
|
|
},
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
const cloudProviderClusterRole = new k8s.rbac.v1.ClusterRole(
|
|
"kube-vip-cloud-controller-role",
|
|
{
|
|
metadata: {
|
|
name: "system:kube-vip-cloud-controller-role",
|
|
annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" },
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: ["coordination.k8s.io"],
|
|
resources: ["leases"],
|
|
verbs: ["get", "create", "update", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: [
|
|
"configmaps",
|
|
"endpoints",
|
|
"events",
|
|
"services/status",
|
|
"leases",
|
|
],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes", "services"],
|
|
verbs: ["list", "get", "watch", "update"],
|
|
},
|
|
],
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
const cloudProviderClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding(
|
|
"kube-vip-cloud-controller-binding",
|
|
{
|
|
metadata: { name: "system:kube-vip-cloud-controller-binding" },
|
|
roleRef: {
|
|
apiGroup: "rbac.authorization.k8s.io",
|
|
kind: "ClusterRole",
|
|
name: cloudProviderClusterRole.metadata.name,
|
|
},
|
|
subjects: [
|
|
{
|
|
kind: "ServiceAccount",
|
|
name: cloudProviderServiceAccount.metadata.name,
|
|
namespace: kubeVipNamespace,
|
|
},
|
|
],
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
const kubeVipCloudProviderLabels = {
|
|
app: "kube-vip",
|
|
component: "kube-vip-cloud-provider",
|
|
};
|
|
|
|
// IP pool the cloud-provider allocates from — scoped to exactly the one VIP we
|
|
// want, so it can never hand the address to some other LoadBalancer Service.
|
|
const kubeVipPool = new k8s.core.v1.ConfigMap(
|
|
"kube-vip-pool",
|
|
{
|
|
metadata: { name: "kubevip", namespace: kubeVipNamespace },
|
|
data: {
|
|
"range-global": pulumi.interpolate`${kubeVipAddress}-${kubeVipAddress}`,
|
|
},
|
|
},
|
|
opts(),
|
|
);
|
|
|
|
const kubeVipCloudProvider = new k8s.apps.v1.Deployment(
|
|
"kube-vip-cloud-provider",
|
|
{
|
|
metadata: { name: "kube-vip-cloud-provider", namespace: kubeVipNamespace },
|
|
spec: {
|
|
replicas: 1,
|
|
selector: {
|
|
matchLabels: kubeVipCloudProviderLabels,
|
|
},
|
|
template: {
|
|
metadata: {
|
|
labels: kubeVipCloudProviderLabels,
|
|
},
|
|
spec: {
|
|
serviceAccountName: cloudProviderServiceAccount.metadata.name,
|
|
tolerations: controlPlaneTolerations,
|
|
containers: [
|
|
{
|
|
name: "kube-vip-cloud-provider",
|
|
image: "ghcr.io/kube-vip/kube-vip-cloud-provider:v0.0.12",
|
|
imagePullPolicy: "Always",
|
|
command: [
|
|
"/kube-vip-cloud-provider",
|
|
"--leader-elect-resource-name=kube-vip-cloud-controller",
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
opts({ dependsOn: [cloudProviderClusterRoleBinding, kubeVipPool] }),
|
|
);
|
|
|
|
new k8s.core.v1.ServicePatch(
|
|
"traefik-vip",
|
|
{
|
|
metadata: {
|
|
name: "traefik",
|
|
namespace: kubeVipNamespace,
|
|
annotations: { "kube-vip.io/loadbalancerIPs": kubeVipAddress },
|
|
},
|
|
spec: { loadBalancerIP: kubeVipAddress },
|
|
},
|
|
opts({ dependsOn: [kubeVip, kubeVipCloudProvider] }),
|
|
);
|