Merge pull request 'added kubevip'

02-k8s-bootstrap/index.ts

@@ -110,7 +110,7 @@ const installMaster1 = new command.remote.Command(
   "install-k3s-master-1",
   {
     connection: conn(master1Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --node-name k3s-master-1`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --cluster-init --tls-san ${master1Ip} --disable servicelb --node-name k3s-master-1`,
     triggers: [master1VmId],
   },
   { dependsOn: [waitMaster1Ssh] },
@@ -154,7 +154,7 @@ const joinMaster2 = new command.remote.Command(
   "join-k3s-master-2",
   {
     connection: conn(master2Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-2`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-2`,
     triggers: [master2VmId],
   },
   { dependsOn: [waitMaster2Ssh] },
@@ -164,7 +164,7 @@ const joinMaster3 = new command.remote.Command(
   "join-k3s-master-3",
   {
     connection: conn(master3Ip),
-    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --node-name k3s-master-3`,
+    create: pulumi.interpolate`curl -sfL https://get.k3s.io | sudo K3S_TOKEN='${k3sToken}' sh -s - server --server https://${master1Ip}:6443 --disable servicelb --node-name k3s-master-3`,
     triggers: [master3VmId],
   },
   { dependsOn: [waitMaster3Ssh, joinMaster2] },

03-k8s-infra/index.ts

@@ -12,6 +12,8 @@ const truenasHost = config.requireSecret("truenasHost");
 const truenasNfsPath = config.requireSecret("truenasNfsPath");
 const cloudflareToken = config.requireSecret("cloudflareApiToken");
 const letsencryptEmail = config.requireSecret("letsencryptEmail");
+const kubeVipAddress = config.requireSecret("kubeVipAddress");
+const kubeVipInterface = config.require("kubeVipInterface");
 
 const k8sProvider = new k8s.Provider("k3s", { kubeconfig });
 const opts = (extras?: pulumi.ResourceOptions): pulumi.ResourceOptions => ({
@@ -134,4 +136,247 @@ new k8s.helm.v3.Release(
   opts(),
 );
 
+// ── kube-vip ────────────
+
+const kubeVipNamespace = "kube-system";
+const controlPlaneTolerations = [
+  { key: "node-role.kubernetes.io/master", effect: "NoSchedule" },
+  { key: "node-role.kubernetes.io/control-plane", effect: "NoSchedule" },
+];
+
+const kubeVipServiceAccount = new k8s.core.v1.ServiceAccount(
+  "kube-vip",
+  { metadata: { name: "kube-vip", namespace: kubeVipNamespace } },
+  opts(),
+);
+
+const kubeVipClusterRole = new k8s.rbac.v1.ClusterRole(
+  "kube-vip-role",
+  {
+    metadata: {
+      name: "system:kube-vip-role",
+      annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" },
+    },
+    rules: [
+      { apiGroups: [""], resources: ["services/status"], verbs: ["update"] },
+      {
+        apiGroups: [""],
+        resources: ["services", "endpoints"],
+        verbs: ["list", "get", "watch", "update"],
+      },
+      {
+        apiGroups: [""],
+        resources: ["nodes"],
+        verbs: ["list", "get", "watch", "update", "patch"],
+      },
+      {
+        apiGroups: ["coordination.k8s.io"],
+        resources: ["leases"],
+        verbs: ["list", "get", "watch", "update", "create"],
+      },
+      {
+        apiGroups: ["discovery.k8s.io"],
+        resources: ["endpointslices"],
+        verbs: ["list", "get", "watch", "update"],
+      },
+      { apiGroups: [""], resources: ["pods"], verbs: ["list"] },
+    ],
+  },
+  opts(),
+);
+
+const kubeVipClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding(
+  "kube-vip-binding",
+  {
+    metadata: { name: "system:kube-vip-binding" },
+    roleRef: {
+      apiGroup: "rbac.authorization.k8s.io",
+      kind: "ClusterRole",
+      name: kubeVipClusterRole.metadata.name,
+    },
+    subjects: [
+      {
+        kind: "ServiceAccount",
+        name: kubeVipServiceAccount.metadata.name,
+        namespace: kubeVipNamespace,
+      },
+    ],
+  },
+  opts(),
+);
+
+const kubeVipLabels = { name: "kube-vip-ds" };
+
+const kubeVip = new k8s.apps.v1.DaemonSet(
+  "kube-vip",
+  {
+    metadata: { name: "kube-vip-ds", namespace: kubeVipNamespace },
+    spec: {
+      selector: { matchLabels: kubeVipLabels },
+      template: {
+        metadata: { labels: kubeVipLabels },
+        spec: {
+          serviceAccountName: kubeVipServiceAccount.metadata.name,
+          hostNetwork: true,
+          tolerations: [
+            ...controlPlaneTolerations.map((t) => ({
+              ...t,
+              operator: "Exists",
+            })),
+            { effect: "NoExecute", operator: "Exists" },
+          ],
+          containers: [
+            {
+              name: "kube-vip",
+              image: "ghcr.io/kube-vip/kube-vip:v1.2.0",
+              imagePullPolicy: "Always",
+              args: ["manager"],
+              env: [
+                { name: "vip_arp", value: "true" },
+                { name: "vip_interface", value: kubeVipInterface },
+                { name: "vip_subnet", value: "32" },
+                { name: "cp_enable", value: "false" },
+                { name: "svc_enable", value: "true" },
+                { name: "svc_election", value: "true" },
+                { name: "vip_leaderelection", value: "true" },
+              ],
+              securityContext: {
+                capabilities: { add: ["NET_ADMIN", "NET_RAW"] },
+              },
+            },
+          ],
+        },
+      },
+    },
+  },
+  opts({ dependsOn: [kubeVipClusterRoleBinding] }),
+);
+
+// -- kube-vip-cloud-provider: allocates LoadBalancer IPs for Services on bare-metal
+// k3s (replacing the disabled ServiceLB). The DaemonSet above only *announces* IPs
+// via ARP; this controller is what actually assigns them to Service.status.
+
+const cloudProviderServiceAccount = new k8s.core.v1.ServiceAccount(
+  "kube-vip-cloud-controller",
+  {
+    metadata: {
+      name: "kube-vip-cloud-controller",
+      namespace: kubeVipNamespace,
+    },
+  },
+  opts(),
+);
+
+const cloudProviderClusterRole = new k8s.rbac.v1.ClusterRole(
+  "kube-vip-cloud-controller-role",
+  {
+    metadata: {
+      name: "system:kube-vip-cloud-controller-role",
+      annotations: { "rbac.authorization.kubernetes.io/autoupdate": "true" },
+    },
+    rules: [
+      {
+        apiGroups: ["coordination.k8s.io"],
+        resources: ["leases"],
+        verbs: ["get", "create", "update", "list", "watch"],
+      },
+      {
+        apiGroups: [""],
+        resources: [
+          "configmaps",
+          "endpoints",
+          "events",
+          "services/status",
+          "leases",
+        ],
+        verbs: ["*"],
+      },
+      {
+        apiGroups: [""],
+        resources: ["nodes", "services"],
+        verbs: ["list", "get", "watch", "update"],
+      },
+    ],
+  },
+  opts(),
+);
+
+const cloudProviderClusterRoleBinding = new k8s.rbac.v1.ClusterRoleBinding(
+  "kube-vip-cloud-controller-binding",
+  {
+    metadata: { name: "system:kube-vip-cloud-controller-binding" },
+    roleRef: {
+      apiGroup: "rbac.authorization.k8s.io",
+      kind: "ClusterRole",
+      name: cloudProviderClusterRole.metadata.name,
+    },
+    subjects: [
+      {
+        kind: "ServiceAccount",
+        name: cloudProviderServiceAccount.metadata.name,
+        namespace: kubeVipNamespace,
+      },
+    ],
+  },
+  opts(),
+);
+
+// IP pool the cloud-provider allocates from — scoped to exactly the one VIP we
+// want, so it can never hand the address to some other LoadBalancer Service.
+const kubeVipPool = new k8s.core.v1.ConfigMap(
+  "kube-vip-pool",
+  {
+    metadata: { name: "kubevip", namespace: kubeVipNamespace },
+    data: { "range-global": pulumi.interpolate`${kubeVipAddress}-${kubeVipAddress}` },
+  },
+  opts(),
+);
+
+const kubeVipCloudProvider = new k8s.apps.v1.Deployment(
+  "kube-vip-cloud-provider",
+  {
+    metadata: { name: "kube-vip-cloud-provider", namespace: kubeVipNamespace },
+    spec: {
+      replicas: 1,
+      selector: {
+        matchLabels: { app: "kube-vip", component: "kube-vip-cloud-provider" },
+      },
+      template: {
+        metadata: {
+          labels: { app: "kube-vip", component: "kube-vip-cloud-provider" },
+        },
+        spec: {
+          serviceAccountName: cloudProviderServiceAccount.metadata.name,
+          tolerations: controlPlaneTolerations,
+          containers: [
+            {
+              name: "kube-vip-cloud-provider",
+              image: "ghcr.io/kube-vip/kube-vip-cloud-provider:v0.0.12",
+              imagePullPolicy: "Always",
+              command: [
+                "/kube-vip-cloud-provider",
+                "--leader-elect-resource-name=kube-vip-cloud-controller",
+              ],
+            },
+          ],
+        },
+      },
+    },
+  },
+  opts({ dependsOn: [cloudProviderClusterRoleBinding, kubeVipPool] }),
+);
+
+new k8s.core.v1.ServicePatch(
+  "traefik-vip",
+  {
+    metadata: {
+      name: "traefik",
+      namespace: kubeVipNamespace,
+      annotations: { "kube-vip.io/loadbalancerIPs": kubeVipAddress },
+    },
+    spec: { loadBalancerIP: kubeVipAddress },
+  },
+  opts({ dependsOn: [kubeVip, kubeVipCloudProvider] }),
+);
+
 export const storageClass = "truenas-nfs";