import * as pulumi from "@pulumi/pulumi"; import * as k8s from "@pulumi/kubernetes"; const config = new pulumi.Config(); const bootstrapRef = new pulumi.StackReference( `${pulumi.getOrganization()}/k8s-bootstrap/dev`, ); const kubeconfig = bootstrapRef.requireOutput("kubeconfig"); export const nfsStorageClass = "truenas-nfs"; const truenasHost = config.requireSecret("truenasHost"); const truenasNfsPath = config.requireSecret("truenasNfsPath"); const cloudflareToken = config.requireSecret("cloudflareApiToken"); const letsencryptEmail = config.requireSecret("letsencryptEmail"); const kubeVipAddress = config.requireSecret("kubeVipAddress"); const kubeVipInterface = config.require("kubeVipInterface"); const k8sProvider = new k8s.Provider("k3s", { kubeconfig }); const opts = (extras?: pulumi.ResourceOptions): pulumi.ResourceOptions => ({ provider: k8sProvider, ...extras, }); // ── NFS CSI Driver ──────────────────────────────────────────────────────── const nfsCsiDriver = new k8s.helm.v3.Release( "nfs-csi-driver", { name: "csi-driver-nfs", chart: "csi-driver-nfs", repositoryOpts: { repo: "https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts", }, namespace: "kube-system", version: "4.12.0", values: { kubeletDir: "/var/lib/kubelet", }, }, opts(), ); new k8s.storage.v1.StorageClass( nfsStorageClass, { metadata: { name: nfsStorageClass }, provisioner: "nfs.csi.k8s.io", parameters: { server: truenasHost, share: truenasNfsPath, mountPermissions: "0", }, reclaimPolicy: "Retain", volumeBindingMode: "Immediate", allowVolumeExpansion: true, mountOptions: ["nfsvers=4.1"], }, opts({ dependsOn: [nfsCsiDriver] }), ); // ── cert-manager ────────────────────────────────────────────────────────── const certManager = new k8s.helm.v3.Release( "cert-manager", { name: "cert-manager", chart: "cert-manager", repositoryOpts: { repo: "https://charts.jetstack.io" }, namespace: "cert-manager", createNamespace: true, timeout: 600, values: { crds: { enabled: true }, }, }, opts(), ); // ── Cloudflare token secret + ClusterIssuer ─────────────────────────────── const cfSecret = new k8s.core.v1.Secret( "cloudflare-token", { metadata: { name: "cloudflare-api-token", namespace: "cert-manager" }, stringData: { "api-token": cloudflareToken }, }, opts({ dependsOn: [certManager] }), ); new k8s.apiextensions.CustomResource( "letsencrypt-prod", { apiVersion: "cert-manager.io/v1", kind: "ClusterIssuer", metadata: { name: "letsencrypt-prod" }, spec: { acme: { server: "https://acme-v02.api.letsencrypt.org/directory", email: letsencryptEmail, privateKeySecretRef: { name: "letsencrypt-prod-account-key" }, solvers: [ { dns01: { cloudflare: { apiTokenSecretRef: { name: cfSecret.metadata.name, key: "api-token", }, }, }, }, ], }, }, }, opts({ dependsOn: [certManager, cfSecret] }), ); // ── Longhorn ────────────────────────────────────────────────────────────── new k8s.helm.v3.Release( "longhorn", { name: "longhorn", chart: "longhorn", repositoryOpts: { repo: "https://charts.longhorn.io" }, namespace: "longhorn-system", createNamespace: true, version: "1.12.0", values: { defaultSettings: { defaultReplicaCount: 3, }, }, }, opts(), ); // ── kube-vip ──────────── const kubeVipNamespace = "kube-system"; const controlPlaneTolerations = [ { key: "node-role.kubernetes.io/master", effect: "NoSchedule" }, { key: "node-role.kubernetes.io/control-plane", effect: "NoSchedule" }, ]; const kubeVipServiceAccount = new k8s.core.v1.ServiceAccount( "kube-vip", { metadata: { name: "kube-vip", namespace: kubeVipNamespace } }, opts(), ); const kubeVipClusterRole = new k8s.rbac.v1.ClusterRole( "kube-vip-role", { metadata: { name: "system:kube-vip-role", annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" }, }, rules: [ { apiGroups: [""], resources: ["services/status"], verbs: ["update"] }, { apiGroups: [""], resources: ["services", "endpoints"], verbs: ["list", "get", "watch", "update"], }, { apiGroups: [""], resources: ["nodes"], verbs: ["list", "get", "watch", "update", "patch"], }, { apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["list", "get", "watch", "update", "create"], }, { apiGroups: ["discovery.k8s.io"], resources: ["endpointslices"], verbs: ["list", "get", "watch", "update"], }, { apiGroups: [""], resources: ["pods"], verbs: ["list"] }, ], }, opts(), ); const kubeVipClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding( "kube-vip-binding", { metadata: { name: "system:kube-vip-binding" }, roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: kubeVipClusterRole.metadata.name, }, subjects: [ { kind: "ServiceAccount", name: kubeVipServiceAccount.metadata.name, namespace: kubeVipNamespace, }, ], }, opts(), ); const kubeVipLabels = { name: "kube-vip-ds" }; const kubeVip = new k8s.apps.v1.DaemonSet( "kube-vip", { metadata: { name: "kube-vip-ds", namespace: kubeVipNamespace }, spec: { selector: { matchLabels: kubeVipLabels }, template: { metadata: { labels: kubeVipLabels }, spec: { serviceAccountName: kubeVipServiceAccount.metadata.name, hostNetwork: true, tolerations: [ ...controlPlaneTolerations.map((t) => ({ ...t, operator: "Exists", })), { effect: "NoExecute", operator: "Exists" }, ], containers: [ { name: "kube-vip", image: "ghcr.io/kube-vip/kube-vip:v1.2.0", imagePullPolicy: "Always", args: ["manager"], env: [ { name: "vip_arp", value: "true" }, { name: "vip_interface", value: kubeVipInterface }, { name: "vip_subnet", value: "32" }, { name: "cp_enable", value: "false" }, { name: "svc_enable", value: "true" }, { name: "svc_election", value: "true" }, { name: "vip_leaderelection", value: "true" }, ], securityContext: { capabilities: { add: ["NET_ADMIN", "NET_RAW"] }, }, }, ], }, }, }, }, opts({ dependsOn: [kubeVipClusterRoleBinding] }), ); // -- kube-vip-cloud-provider: allocates LoadBalancer IPs for Services on bare-metal // k3s (replacing the disabled ServiceLB). The DaemonSet above only *announces* IPs // via ARP; this controller is what actually assigns them to Service.status. const cloudProviderServiceAccount = new k8s.core.v1.ServiceAccount( "kube-vip-cloud-controller", { metadata: { name: "kube-vip-cloud-controller", namespace: kubeVipNamespace, }, }, opts(), ); const cloudProviderClusterRole = new k8s.rbac.v1.ClusterRole( "kube-vip-cloud-controller-role", { metadata: { name: "system:kube-vip-cloud-controller-role", annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" }, }, rules: [ { apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["get", "create", "update", "list", "watch"], }, { apiGroups: [""], resources: [ "configmaps", "endpoints", "events", "services/status", "leases", ], verbs: ["*"], }, { apiGroups: [""], resources: ["nodes", "services"], verbs: ["list", "get", "watch", "update"], }, ], }, opts(), ); const cloudProviderClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding( "kube-vip-cloud-controller-binding", { metadata: { name: "system:kube-vip-cloud-controller-binding" }, roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: cloudProviderClusterRole.metadata.name, }, subjects: [ { kind: "ServiceAccount", name: cloudProviderServiceAccount.metadata.name, namespace: kubeVipNamespace, }, ], }, opts(), ); const kubeVipCloudProviderLabels = { app: "kube-vip", component: "kube-vip-cloud-provider", }; // IP pool the cloud-provider allocates from — scoped to exactly the one VIP we // want, so it can never hand the address to some other LoadBalancer Service. const kubeVipPool = new k8s.core.v1.ConfigMap( "kube-vip-pool", { metadata: { name: "kubevip", namespace: kubeVipNamespace }, data: { "range-global": pulumi.interpolate`${kubeVipAddress}-${kubeVipAddress}` }, }, opts(), ); const kubeVipCloudProvider = new k8s.apps.v1.Deployment( "kube-vip-cloud-provider", { metadata: { name: "kube-vip-cloud-provider", namespace: kubeVipNamespace }, spec: { replicas: 1, selector: { matchLabels: kubeVipCloudProviderLabels, }, template: { metadata: { labels: kubeVipCloudProviderLabels, }, spec: { serviceAccountName: cloudProviderServiceAccount.metadata.name, tolerations: controlPlaneTolerations, containers: [ { name: "kube-vip-cloud-provider", image: "ghcr.io/kube-vip/kube-vip-cloud-provider:v0.0.12", imagePullPolicy: "Always", command: [ "/kube-vip-cloud-provider", "--leader-elect-resource-name=kube-vip-cloud-controller", ], }, ], }, }, }, }, opts({ dependsOn: [cloudProviderClusterRoleBinding, kubeVipPool] }), ); new k8s.core.v1.ServicePatch( "traefik-vip", { metadata: { name: "traefik", namespace: kubeVipNamespace, annotations: { "kube-vip.io/loadbalancerIPs": kubeVipAddress }, }, spec: { loadBalancerIP: kubeVipAddress }, }, opts({ dependsOn: [kubeVip, kubeVipCloudProvider] }), );