[PRODENG-3744] Use KDD datastore by default

ansible/mke-post-install-playbook.yml

@@ -46,3 +46,27 @@
   tasks:
     - name: Install machine-config-controller
       ansible.builtin.include_tasks: tasks/machine-config-controller-tasks.yml
+
+# Runs last, deliberately: cluster-upgrade-controller/machine-config-controller
+# (and SUC, above) each apply CRDs/namespaces/Deployments of their own, and
+# doing so was observed to trigger MKE re-syncing its baseline kube-system
+# manifests — reverting this patch minutes after it was applied and long
+# after its own hold-and-reassert window (tasks/calico-kdd-patch-tasks.yml)
+# had already declared success. Running this after every other mutating
+# step in this playbook means nothing later in the same run can re-trigger
+# that reconciliation and undo the fix.
+- name: Patch calico-kube-controllers service account for KDD
+  hosts: localhost
+  gather_facts: false
+  vars_files:
+    - vars/common-vars.yml
+    - vars/mke-creds.yml
+  vars:
+    # mke_url lives in inventory all.vars; pull it from a real host's hostvars
+    # since localhost is not part of the inventory group.
+    mke_url: "{{ hostvars[groups['managers'][0]]['mke_url'] }}"
+    bundle_dest: "{{ playbook_dir }}/mke-bundle"
+  tasks:
+    - name: Patch calico-kube-controllers service account
+      ansible.builtin.include_tasks: tasks/calico-kdd-patch-tasks.yml
+      when: calico_datastore_type_kdd | bool

ansible/tasks/calico-kdd-patch-tasks.yml

@@ -0,0 +1,50 @@
+# Patches the calico-kube-controllers Deployment in kube-system to run
+# under the calico-kube-controllers ServiceAccount instead of calico-node.
+#
+# Known MKE3 issue: when Calico is installed with the Kubernetes Datastore
+# (KDD) backend (--calico-datastore-type-kdd at `mke install` time — see
+# calico_datastore_type_kdd in vars/common-vars.yml), MKE brings up the
+# calico-kube-controllers Deployment still wired to the calico-node
+# ServiceAccount. calico-node's RBAC is scoped for the per-node agent, not
+# the cluster-wide controller, so calico-kube-controllers cannot sync from
+# the Kubernetes API (CrashLoopBackOff / RBAC "forbidden" errors in its
+# logs) until it runs as the calico-kube-controllers ServiceAccount that MKE
+# also creates but never wires up.
+#
+# Installing cluster-upgrade-controller/machine-config-controller/SUC each
+# apply their own CRDs/namespaces/Deployments, which was observed to trigger
+# MKE to re-sync its baseline kube-system manifests — reverting this patch
+# if it ran before them. mke-post-install-playbook.yml therefore runs this
+# task last, after every other mutating step, so nothing later in the same
+# run can undo it.
+#
+# Required vars (inherited from the enclosing play):
+#   admin_user  – MKE administrator username       (from vars/mke-creds.yml)
+#   admin_pass  – MKE administrator password       (from vars/mke-creds.yml)
+#   mke_url     – MKE load-balancer IP/hostname    (from inventory all.vars)
+#   bundle_dest – local directory for the bundle   (default: playbook_dir/mke-bundle)
+#
+# Ref: https://docs.mirantis.com/mke/3.7/cli-ref/mke-cli-install.html
+
+- name: Fetch MKE client bundle
+  ansible.builtin.include_tasks: tasks/mke-client-bundle-tasks.yml
+
+- name: Patch calico-kube-controllers service account
+  ansible.builtin.command:
+    argv:
+      - kubectl
+      - patch
+      - deployment
+      - calico-kube-controllers
+      - -n
+      - kube-system
+      - --kubeconfig
+      - "{{ bundle_dest }}/kube.yml"
+      - --type
+      - merge
+      - --patch
+      - '{"spec":{"template":{"spec":{"serviceAccountName":"calico-kube-controllers"}}}}'
+  delegate_to: localhost
+  run_once: true
+  register: _calico_kdd_patch
+  changed_when: "'patched' in _calico_kdd_patch.stdout"

ansible/tasks/mke-install-tasks.yml

@@ -18,6 +18,12 @@
   when: mke_external_certs_enabled | bool
   run_once: true
 
+- name: Append the --calico-datastore-type-kdd MKE flag to the list
+  ansible.builtin.set_fact:
+    mke_install_flags: "{{ mke_install_flags + ['--calico-datastore-type-kdd'] }}"
+  when: calico_datastore_type_kdd | bool
+  run_once: true
+
 - name: Get target MKE version
   ansible.builtin.include_tasks: tasks/helpers/get_target_mke_version.yml
 

ansible/vars/common-vars.yml

@@ -23,6 +23,22 @@ mke_install_flags:
     "--force-minimums",
   ]
 
+# When true, `mke install` is run with --calico-datastore-type-kdd, switching
+# Calico's datastore backend from etcd to the Kubernetes API (KDD).
+#
+# Known MKE3 issue: with KDD enabled, MKE brings up the
+# calico-kube-controllers Deployment (kube-system) still wired to the
+# calico-node ServiceAccount instead of the calico-kube-controllers
+# ServiceAccount it also creates — calico-node's RBAC is scoped for the node
+# agent, not the controller, so the pod cannot sync from the Kubernetes API.
+# tasks/calico-kdd-patch-tasks.yml (run from mke-post-install-playbook.yml)
+# patches the Deployment's serviceAccountName once the cluster is up. Keep
+# this var and that task's `when` in sync if you flip this.
+#
+# NOTE: MKE team engineers don't recommend to use KDD by default, because
+# it is not yet fully hardened from a product perspective
+calico_datastore_type_kdd: false
+
 # Use your own TLS certificate for the MKE web server/API instead of the
 # self-signed certificate MKE generates. Applied only at install time, via
 # `--external-server-cert` (tasks/mke-external-certs-tasks.yml populates the