CP-43356: convert KUTTL command:+args: steps to valid top-level commands:

.github/workflows/test-matrix-k8s.yaml

@@ -122,6 +122,13 @@ jobs:
           command: |
             make install-tools V=1
 
+      - name: Log in to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Run Complete Chart Tests
         env:
           CLOUDZERO_DEV_API_KEY: ${{ secrets.CLOUDZERO_DEV_API_KEY }}
@@ -134,8 +141,29 @@ jobs:
           # Create cluster configuration files for this test run
           cp clusters/kind.yaml "clusters/$CLUSTER_NAME.yaml"
           cp clusters/kind-overrides.yaml "clusters/$CLUSTER_NAME-overrides.yaml"
-          # Run the complete test workflow using Kind
-          make kind-test "CLUSTER_NAME=$CLUSTER_NAME" V=1
+
+          # The chart (and the alloy-clustered-test suite) install the agent image
+          # at <prod-repo>:dev-<sha>, which is NOT published for a PR. Pull the
+          # freshly-built (untested) image for this matrix arch, retag it as the
+          # tag the chart installs, and kind-load it into the node so pods can start
+          # (chart imagePullPolicy is IfNotPresent). Pull single-arch: kind cannot
+          # load a multi-arch manifest list.
+          SRC="${{ inputs.image-repo }}/${{ inputs.image-path }}:${{ inputs.image-tag }}"
+          DST="ghcr.io/cloudzero/cloudzero-agent:dev-$(git rev-parse HEAD)"
+          docker pull --platform "${{ matrix.kver.platform }}" "$SRC"
+          docker tag "$SRC" "$DST"
+
+          # Run the test workflow phases explicitly so the image is loaded between
+          # cluster creation and chart install (the monolithic `kind-test` target
+          # does not expose that seam).
+          test_status=0
+          make kind-up "CLUSTER_NAME=$CLUSTER_NAME" V=1
+          make kind-load "CLUSTER_NAME=$CLUSTER_NAME" LOAD_IMAGE="$DST" V=1
+          make helm-install-current helm-wait "CLUSTER_NAME=$CLUSTER_NAME" V=1
+          make helm-test-kuttl "CLUSTER_NAME=$CLUSTER_NAME" V=1 || test_status=$?
+          make helm-uninstall "CLUSTER_NAME=$CLUSTER_NAME" V=1 || true
+          make kind-down "CLUSTER_NAME=$CLUSTER_NAME" V=1 || true
+          exit $test_status
 
       - name: Set version output
         id: set-output
@@ -250,6 +278,13 @@ jobs:
           echo "Contents:"
           cat "$EXTRA_OVERRIDES_FILE"
 
+      - name: Log in to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Run Chart Tests with Specialized Configuration
         env:
           CLOUDZERO_DEV_API_KEY: ${{ secrets.CLOUDZERO_DEV_API_KEY }}
@@ -263,8 +298,25 @@ jobs:
           # Create cluster configuration files for this test run
           cp clusters/kind.yaml "clusters/$CLUSTER_NAME.yaml"
           cp clusters/kind-overrides.yaml "clusters/$CLUSTER_NAME-overrides.yaml"
-          # Run the complete test workflow using Kind with extra overrides
-          make kind-test "CLUSTER_NAME=$CLUSTER_NAME" "HELM_EXTRA_OVERRIDES=$HELM_EXTRA_OVERRIDES" V=1
+
+          # Make the freshly-built (untested) agent image available to the kind
+          # node under the tag the chart installs (see the version-matrix job for
+          # the full rationale). Pull single-arch so kind can load it.
+          SRC="${{ inputs.image-repo }}/${{ inputs.image-path }}:${{ inputs.image-tag }}"
+          DST="ghcr.io/cloudzero/cloudzero-agent:dev-$(git rev-parse HEAD)"
+          docker pull --platform linux/amd64 "$SRC"
+          docker tag "$SRC" "$DST"
+
+          # Run the test workflow phases explicitly so the image is loaded between
+          # cluster creation and chart install.
+          test_status=0
+          make kind-up "CLUSTER_NAME=$CLUSTER_NAME" V=1
+          make kind-load "CLUSTER_NAME=$CLUSTER_NAME" LOAD_IMAGE="$DST" V=1
+          make helm-install-current helm-wait "CLUSTER_NAME=$CLUSTER_NAME" "HELM_EXTRA_OVERRIDES=$HELM_EXTRA_OVERRIDES" V=1
+          make helm-test-kuttl "CLUSTER_NAME=$CLUSTER_NAME" "HELM_EXTRA_OVERRIDES=$HELM_EXTRA_OVERRIDES" V=1 || test_status=$?
+          make helm-uninstall "CLUSTER_NAME=$CLUSTER_NAME" "HELM_EXTRA_OVERRIDES=$HELM_EXTRA_OVERRIDES" V=1 || true
+          make kind-down "CLUSTER_NAME=$CLUSTER_NAME" V=1 || true
+          exit $test_status
 
       - name: Set config output
         id: set-output

Makefile

@@ -430,6 +430,16 @@ CLUSTER_NAME              ?= kind
 kind-up: # Create kind cluster for testing
 kind-up: tests/kuttl/kubeconfig
 
+# kind-load - Load a locally-available docker image into the kind cluster's node
+# so pods can use it without pulling from a registry (chart imagePullPolicy is
+# IfNotPresent). Used by CI to make the freshly-built (untested) agent image
+# available under the tag the chart installs. Set LOAD_IMAGE to the image ref.
+.PHONY: kind-load
+kind-load: ## Load LOAD_IMAGE into the kind cluster (uses CLUSTER_NAME)
+	$(if $(LOAD_IMAGE),,$(error LOAD_IMAGE must be set, e.g. make kind-load LOAD_IMAGE=repo/image:tag))
+	$(call LOG,KIND,load $(LOAD_IMAGE) into $(CLUSTER_NAME))
+	$(Q)$(KIND) load docker-image "$(LOAD_IMAGE)" --name $(CLUSTER_NAME)
+
 .PHONY: kind-down
 kind-down: ## Delete kind cluster and cleanup
 	$(call LOG,KIND,delete cluster $(CLUSTER_NAME))

tests/kuttl/alloy-clustered-test/steps/01-install-clustered-chart.yaml

@@ -2,21 +2,32 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 metadata:
   name: install-clustered-chart
-spec:
-  commands:
-    # Install the chart in clustered mode as a separate release so we do not
-    # disturb the default cz-agent release other KUTTL suites run against.
-    # The dev image tag mirrors the `make helm-install-current` target.
-    - command: sh
-      args:
-        - "-c"
-        - |
-          set -e
-          DEV_TAG="dev-$(git rev-parse HEAD)"
-          .tools/bin/helm upgrade --install cz-alloy-ct ./helm \
-            --namespace cz-alloy-clustered-test \
-            --create-namespace \
-            --values clusters/kind-overrides.yaml \
-            --set components.agent.mode=clustered \
-            --set components.agent.image.tag="$DEV_TAG" \
-            --wait --timeout=3m
+# kuttl command steps use top-level `commands:` with `script:` (a shell-script
+# string). kuttl's TestStep has no `spec` field and its Command has no `args`
+# field (`command` is a single string), so the previous `spec:`+`command:`+`args:`
+# shape silently no-opped. kuttl also runs scripts with CWD = this step directory,
+# so we walk up to the repo root before using repo-relative paths
+# (.tools/bin/helm, ./helm, clusters/).
+commands:
+  # Install the chart in clustered mode as a separate release so we do not
+  # disturb the default release other kuttl suites run against.
+  # The dev image tag mirrors the `make helm-install-current` target.
+  #
+  # `timeout:` must exceed `helm --wait --timeout=3m` (180s). kuttl applies a 30s
+  # per-command timeout here (this suite's kuttl-test.yaml sets `spec.timeout`, which
+  # kuttl ignores — the timeout field is top-level — so the 30s default applies), and
+  # that would kill the install mid-wait. The explicit per-command `timeout:` overrides it.
+  - timeout: 240
+    script: |
+      set -e
+      ROOT="$PWD"
+      while [ "$ROOT" != "/" ] && [ ! -f "$ROOT/helm/Chart.yaml" ]; do ROOT=$(dirname "$ROOT"); done
+      cd "$ROOT"
+      DEV_TAG="dev-$(git rev-parse HEAD)"
+      .tools/bin/helm upgrade --install cz-alloy-ct ./helm \
+        --namespace cz-alloy-clustered-test \
+        --create-namespace \
+        --values clusters/kind-overrides.yaml \
+        --set components.agent.mode=clustered \
+        --set components.agent.image.tag="$DEV_TAG" \
+        --wait --timeout=3m

tests/kuttl/alloy-clustered-test/steps/02-verify-alloy-ready.yaml

@@ -2,59 +2,74 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 metadata:
   name: verify-alloy-ready
-spec:
-  commands:
-    # Belt-and-suspenders: step 01's `helm --wait` already blocks on Ready,
-    # but re-assert explicitly here so the failure message points at readiness
-    # rather than at helm timing out.
-    - command: kubectl
-      args:
-        - wait
-        - --for=condition=Ready
-        - pod
-        - -l
-        - app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct
-        - -n
-        - cz-alloy-clustered-test
-        - --timeout=180s
-    # Fail if the Alloy container has restarted at all. The pre-fix failure
-    # is CrashLoopBackOff on first start, so any restart > 0 before Ready
-    # flags the regression even if a later retry somehow succeeded.
-    - command: sh
-      args:
-        - "-c"
-        - |
-          set -e
-          RESTARTS=$(kubectl get pod \
+# kuttl command steps use top-level `commands:` with `script:`. kuttl's TestStep
+# has no `spec` field and its Command has no `args` field, so the previous
+# `spec:`+`command:`+`args:` shape silently no-opped. These steps are kubectl-only
+# and read KUBECONFIG from the environment, so no repo-root cd is needed.
+commands:
+  # Belt-and-suspenders: step 01's `helm --wait` already blocks on Ready,
+  # but re-assert explicitly here so the failure message points at readiness
+  # rather than at helm timing out.
+  #
+  # Pre-check: `kubectl wait -l <selector>` exits 0 immediately on kubectl <1.27
+  # when no pod yet matches, which would vacuously pass. Poll until at least one
+  # matching pod exists before handing off to kubectl wait.
+  #
+  # `timeout:` must cover the up-to-~60s poll loop + `kubectl wait --timeout=180s`; kuttl's
+  # 30s per-command default (suite spec.timeout is ignored) would otherwise clamp it.
+  - timeout: 300
+    script: |
+      set -e
+      for i in $(seq 1 30); do
+        COUNT=$(kubectl get pod \
+          -l app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct \
+          -n cz-alloy-clustered-test \
+          --no-headers 2>/dev/null | wc -l)
+        if [ "$COUNT" -gt 0 ]; then
+          echo "Alloy (server) pod appeared (attempt $i)"
+          break
+        fi
+        [ "$i" -eq 30 ] && { echo "FAIL: Alloy (server) pod never appeared" >&2; exit 1; }
+        sleep 2
+      done
+      kubectl wait --for=condition=Ready pod \
+        -l app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct \
+        -n cz-alloy-clustered-test \
+        --timeout=180s
+
+  # Fail if the Alloy container has restarted at all. The pre-fix failure
+  # is CrashLoopBackOff on first start, so any restart > 0 before Ready
+  # flags the regression even if a later retry somehow succeeded.
+  - script: |
+      set -e
+      RESTARTS=$(kubectl get pod \
+        -l app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct \
+        -n cz-alloy-clustered-test \
+        -o jsonpath='{.items[*].status.containerStatuses[?(@.name=="cloudzero-agent-alloy")].restartCount}')
+      echo "Alloy container restart count: $RESTARTS"
+      for r in $RESTARTS; do
+        if [ "$r" -ne 0 ]; then
+          echo "FAIL: Alloy container restarted $r times (expected 0)" >&2
+          kubectl describe pod \
             -l app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct \
-            -n cz-alloy-clustered-test \
-            -o jsonpath='{.items[*].status.containerStatuses[?(@.name=="cloudzero-agent-alloy")].restartCount}')
-          echo "Alloy container restart count: $RESTARTS"
-          for r in $RESTARTS; do
-            if [ "$r" -ne 0 ]; then
-              echo "FAIL: Alloy container restarted $r times (expected 0)" >&2
-              kubectl describe pod \
-                -l app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct \
-                -n cz-alloy-clustered-test >&2 || true
-              exit 1
-            fi
-          done
-    # Fail if the specific /tmp regression signature appears in Alloy logs.
-    # Pins this assertion to CP-40307 so unrelated clustered-mode failures
-    # can't silently green-bar.
-    - command: sh
-      args:
-        - "-c"
-        - |
-          set -e
-          LOGS=$(kubectl logs \
-            -l app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct \
-            -n cz-alloy-clustered-test \
-            -c cloudzero-agent-alloy \
-            --all-containers=false \
-            --tail=-1 2>&1 || true)
-          if echo "$LOGS" | grep -q 'mkdir /tmp: permission denied'; then
-            echo "FAIL: Alloy logs contain the CP-40307 regression signature" >&2
-            echo "$LOGS" >&2
-            exit 1
-          fi
+            -n cz-alloy-clustered-test >&2 || true
+          exit 1
+        fi
+      done
+
+  # Fail if the specific /tmp regression signature appears in Alloy logs.
+  # Pins this assertion to CP-40307 so unrelated clustered-mode failures
+  # can't silently green-bar.
+  - script: |
+      set -e
+      LOGS=$(kubectl logs \
+        -l app.kubernetes.io/name=server,app.kubernetes.io/instance=cz-alloy-ct \
+        -n cz-alloy-clustered-test \
+        -c cloudzero-agent-alloy \
+        --all-containers=false \
+        --tail=-1 2>&1 || true)
+      if echo "$LOGS" | grep -q 'mkdir /tmp: permission denied'; then
+        echo "FAIL: Alloy logs contain the CP-40307 regression signature" >&2
+        echo "$LOGS" >&2
+        exit 1
+      fi

tests/kuttl/alloy-clustered-test/steps/99-uninstall.yaml

@@ -2,17 +2,23 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 metadata:
   name: uninstall-clustered-chart
-spec:
-  commands:
-    # Uninstall the dedicated release so subsequent kuttl suites and the
-    # `kind-test` teardown are unaffected. `--ignore-not-found` and `|| true`
-    # keep cleanup best-effort even if an earlier step failed.
-    - command: sh
-      args:
-        - "-c"
-        - |
-          .tools/bin/helm uninstall cz-alloy-ct \
-            --namespace cz-alloy-clustered-test \
-            --ignore-not-found || true
-          kubectl delete namespace cz-alloy-clustered-test \
-            --ignore-not-found --wait=false || true
+# Uses top-level `commands:` with `script:` (kuttl's TestStep has no `spec` field
+# and its Command has no `args` field). Walks up to the repo root before calling
+# .tools/bin/helm, since kuttl runs scripts with CWD = this step dir.
+commands:
+  # Uninstall the dedicated release so subsequent kuttl suites and the
+  # `kind-test` teardown are unaffected. `--ignore-not-found` and `|| true`
+  # keep cleanup best-effort even if an earlier step failed.
+  #
+  # Headroom over the 30s per-command timeout (this suite's kuttl-test.yaml sets
+  # `spec.timeout`, which kuttl ignores, so the 30s default applies) for helm uninstall.
+  - timeout: 120
+    script: |
+      ROOT="$PWD"
+      while [ "$ROOT" != "/" ] && [ ! -f "$ROOT/helm/Chart.yaml" ]; do ROOT=$(dirname "$ROOT"); done
+      cd "$ROOT"
+      .tools/bin/helm uninstall cz-alloy-ct \
+        --namespace cz-alloy-clustered-test \
+        --ignore-not-found || true
+      kubectl delete namespace cz-alloy-clustered-test \
+        --ignore-not-found --wait=false || true