diff --git a/02-k8s-bootstrap/index.ts b/02-k8s-bootstrap/index.ts index d4ae119..1836e69 100644 --- a/02-k8s-bootstrap/index.ts +++ b/02-k8s-bootstrap/index.ts @@ -110,7 +110,7 @@ const installMaster1 = new command.remote.Command( "install-k3s-master-1", { connection: conn(master1Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --disable servicelb --node-name k3s-master-1`, triggers: [master1VmId], }, { dependsOn: [waitMaster1Ssh] }, @@ -154,7 +154,7 @@ const joinMaster2 = new command.remote.Command( "join-k3s-master-2", { connection: conn(master2Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-2`, triggers: [master2VmId], }, { dependsOn: [waitMaster2Ssh] }, @@ -164,7 +164,7 @@ const joinMaster3 = new command.remote.Command( "join-k3s-master-3", { connection: conn(master3Ip), - create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`, + create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-3`, triggers: [master3VmId], }, { dependsOn: [waitMaster3Ssh, joinMaster2] }, diff --git a/03-k8s-infra/index.ts b/03-k8s-infra/index.ts index a600ab9..aaf3ba6 100644 --- a/03-k8s-infra/index.ts +++ b/03-k8s-infra/index.ts @@ -12,6 +12,8 @@ const truenasHost = config.requireSecret("truenasHost"); const truenasNfsPath = config.requireSecret("truenasNfsPath"); const cloudflareToken = config.requireSecret("cloudflareApiToken"); const letsencryptEmail = config.requireSecret("letsencryptEmail"); +const kubeVipAddress = config.requireSecret("kubeVipAddress"); +const kubeVipInterface = config.require("kubeVipInterface"); const k8sProvider = new k8s.Provider("k3s", { kubeconfig }); const opts = (extras?: pulumi.ResourceOptions): pulumi.ResourceOptions => ({ @@ -134,4 +136,247 @@ new k8s.helm.v3.Release( opts(), ); +// ── kube-vip ──────────── + +const kubeVipNamespace = "kube-system"; +const controlPlaneTolerations = [ + { key: "node-role.kubernetes.io/master", effect: "NoSchedule" }, + { key: "node-role.kubernetes.io/control-plane", effect: "NoSchedule" }, +]; + +const kubeVipServiceAccount = new k8s.core.v1.ServiceAccount( + "kube-vip", + { metadata: { name: "kube-vip", namespace: kubeVipNamespace } }, + opts(), +); + +const kubeVipClusterRole = new k8s.rbac.v1.ClusterRole( + "kube-vip-role", + { + metadata: { + name: "system:kube-vip-role", + annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" }, + }, + rules: [ + { apiGroups: [""], resources: ["services/status"], verbs: ["update"] }, + { + apiGroups: [""], + resources: ["services", "endpoints"], + verbs: ["list", "get", "watch", "update"], + }, + { + apiGroups: [""], + resources: ["nodes"], + verbs: ["list", "get", "watch", "update", "patch"], + }, + { + apiGroups: ["coordination.k8s.io"], + resources: ["leases"], + verbs: ["list", "get", "watch", "update", "create"], + }, + { + apiGroups: ["discovery.k8s.io"], + resources: ["endpointslices"], + verbs: ["list", "get", "watch", "update"], + }, + { apiGroups: [""], resources: ["pods"], verbs: ["list"] }, + ], + }, + opts(), +); + +const kubeVipClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding( + "kube-vip-binding", + { + metadata: { name: "system:kube-vip-binding" }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + name: kubeVipClusterRole.metadata.name, + }, + subjects: [ + { + kind: "ServiceAccount", + name: kubeVipServiceAccount.metadata.name, + namespace: kubeVipNamespace, + }, + ], + }, + opts(), +); + +const kubeVipLabels = { name: "kube-vip-ds" }; + +const kubeVip = new k8s.apps.v1.DaemonSet( + "kube-vip", + { + metadata: { name: "kube-vip-ds", namespace: kubeVipNamespace }, + spec: { + selector: { matchLabels: kubeVipLabels }, + template: { + metadata: { labels: kubeVipLabels }, + spec: { + serviceAccountName: kubeVipServiceAccount.metadata.name, + hostNetwork: true, + tolerations: [ + ...controlPlaneTolerations.map((t) => ({ + ...t, + operator: "Exists", + })), + { effect: "NoExecute", operator: "Exists" }, + ], + containers: [ + { + name: "kube-vip", + image: "ghcr.io/kube-vip/kube-vip:v1.2.0", + imagePullPolicy: "Always", + args: ["manager"], + env: [ + { name: "vip_arp", value: "true" }, + { name: "vip_interface", value: kubeVipInterface }, + { name: "vip_subnet", value: "32" }, + { name: "cp_enable", value: "false" }, + { name: "svc_enable", value: "true" }, + { name: "svc_election", value: "true" }, + { name: "vip_leaderelection", value: "true" }, + ], + securityContext: { + capabilities: { add: ["NET_ADMIN", "NET_RAW"] }, + }, + }, + ], + }, + }, + }, + }, + opts({ dependsOn: [kubeVipClusterRoleBinding] }), +); + +// -- kube-vip-cloud-provider: allocates LoadBalancer IPs for Services on bare-metal +// k3s (replacing the disabled ServiceLB). The DaemonSet above only *announces* IPs +// via ARP; this controller is what actually assigns them to Service.status. + +const cloudProviderServiceAccount = new k8s.core.v1.ServiceAccount( + "kube-vip-cloud-controller", + { + metadata: { + name: "kube-vip-cloud-controller", + namespace: kubeVipNamespace, + }, + }, + opts(), +); + +const cloudProviderClusterRole = new k8s.rbac.v1.ClusterRole( + "kube-vip-cloud-controller-role", + { + metadata: { + name: "system:kube-vip-cloud-controller-role", + annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" }, + }, + rules: [ + { + apiGroups: ["coordination.k8s.io"], + resources: ["leases"], + verbs: ["get", "create", "update", "list", "watch"], + }, + { + apiGroups: [""], + resources: [ + "configmaps", + "endpoints", + "events", + "services/status", + "leases", + ], + verbs: ["*"], + }, + { + apiGroups: [""], + resources: ["nodes", "services"], + verbs: ["list", "get", "watch", "update"], + }, + ], + }, + opts(), +); + +const cloudProviderClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding( + "kube-vip-cloud-controller-binding", + { + metadata: { name: "system:kube-vip-cloud-controller-binding" }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + name: cloudProviderClusterRole.metadata.name, + }, + subjects: [ + { + kind: "ServiceAccount", + name: cloudProviderServiceAccount.metadata.name, + namespace: kubeVipNamespace, + }, + ], + }, + opts(), +); + +// IP pool the cloud-provider allocates from — scoped to exactly the one VIP we +// want, so it can never hand the address to some other LoadBalancer Service. +const kubeVipPool = new k8s.core.v1.ConfigMap( + "kube-vip-pool", + { + metadata: { name: "kubevip", namespace: kubeVipNamespace }, + data: { "range-global": pulumi.interpolate`${kubeVipAddress}-${kubeVipAddress}` }, + }, + opts(), +); + +const kubeVipCloudProvider = new k8s.apps.v1.Deployment( + "kube-vip-cloud-provider", + { + metadata: { name: "kube-vip-cloud-provider", namespace: kubeVipNamespace }, + spec: { + replicas: 1, + selector: { + matchLabels: { app: "kube-vip", component: "kube-vip-cloud-provider" }, + }, + template: { + metadata: { + labels: { app: "kube-vip", component: "kube-vip-cloud-provider" }, + }, + spec: { + serviceAccountName: cloudProviderServiceAccount.metadata.name, + tolerations: controlPlaneTolerations, + containers: [ + { + name: "kube-vip-cloud-provider", + image: "ghcr.io/kube-vip/kube-vip-cloud-provider:v0.0.12", + imagePullPolicy: "Always", + command: [ + "/kube-vip-cloud-provider", + "--leader-elect-resource-name=kube-vip-cloud-controller", + ], + }, + ], + }, + }, + }, + }, + opts({ dependsOn: [cloudProviderClusterRoleBinding, kubeVipPool] }), +); + +new k8s.core.v1.ServicePatch( + "traefik-vip", + { + metadata: { + name: "traefik", + namespace: kubeVipNamespace, + annotations: { "kube-vip.io/loadbalancerIPs": kubeVipAddress }, + }, + spec: { loadBalancerIP: kubeVipAddress }, + }, + opts({ dependsOn: [kubeVip, kubeVipCloudProvider] }), +); + export const storageClass = "truenas-nfs";