refactor(config): vigie config and test file definition clarification

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         files: \.go$
 
       - id: check-schema
-        name: schema in sync with DSL types
+        name: schemas in sync with DSL and config types
         entry: make check-schema
         language: system
         pass_filenames: false

Makefile

@@ -44,12 +44,12 @@ pre-commit: ## Run all pre-commit hooks against every file
 changelog: ## Generate a changelog from git history
 	git cliff --output CHANGELOG.md
 
-gen-schema: ## Regenerate pkg/api/schema/v1/testfile.json from internal/dsl types
+gen-schema: ## Regenerate the JSON Schemas from internal/dsl and internal/config types
 	go generate ./pkg/api/schema/v1/...
 
-check-schema: ## Fail if testfile.json is out of sync with `go generate`
+check-schema: ## Fail if the generated JSON Schemas are out of sync with `go generate`
 	go generate ./pkg/api/schema/v1/...
-	git diff --exit-code pkg/api/schema/v1/testfile.json
+	git diff --exit-code pkg/api/schema/v1/testfile.json pkg/api/schema/v1/config.json
 
 release-dry-run: ## Preview a release locally (requires goreleaser in PATH)
 	goreleaser release --snapshot --clean

README.md

@@ -44,13 +44,14 @@ with **no pre-existing cluster**. Both drive the upstream CLI (vigie does not em
 need a container runtime — **docker or podman** — on the host.
 
 vigie resolves the `kind`/`k3d` binary in order: an explicit `--kind-binary` / `--k3d-binary`
-path → `$PATH` → the vigie cache → download. Downloads are opt-in: on an interactive terminal
+path → `test.cluster.<backend>.binary` in `.vigie.yaml` → `$PATH` → the vigie cache → download.
+Downloads are opt-in: on an interactive terminal
 vigie prompts for confirmation; in CI or with piped stdin it never downloads and errors with
 install guidance instead, unless you pass `--download-tools` (or set `VIGIE_AUTO_DOWNLOAD=1`).
 Minimum supported versions: **kind ≥ v0.20.0**, **k3d ≥ v5.4.0**.
 
-Backend-specific provisioning flags go through `testApply.cluster.extraArgs` in `.vigie.yaml`
-(e.g. a kind `--config` for a multi-node topology). Downloaded binaries are statically-linked
+Backend-specific provisioning flags go through `test.cluster.<backend>.extraArgs` in
+`.vigie.yaml` (e.g. a kind `--config` for a multi-node topology). Downloaded binaries are statically-linked
 Go executables that run on NixOS as-is; a `nix profile install kind k3d` is picked up from
 `$PATH` before any download.
 
@@ -77,7 +78,8 @@ go build -o vigie ./cmd/vigie
 
 ## Quick start
 
-Drop test files under `tests/unit/` in your chart:
+Drop test files under `tests/` in your chart — one root for every test file, scanned
+recursively, with sub-directories purely for organisation:
 
 ```yaml
 # mychart/tests/unit/deployment_test.yaml
@@ -124,8 +126,9 @@ Tests: 2 total, 2 passed (2ms total test time)
 ```
 
 A complete, realistic example chart lives in
-[`testdata/charts/basic`](./testdata/charts/basic) — its `tests/unit/` suite exercises the
-full matcher library, `matrix`/`cases`, helper (`call:`) tests, and snapshots.
+[`testdata/charts/basic`](./testdata/charts/basic) — its `tests/` root exercises the full
+matcher library, `matrix`/`cases`, helper (`call:`) tests, and snapshots, plus apply-tier
+suites with dependencies and live matchers.
 
 ---
 
@@ -163,6 +166,7 @@ under a `validate:` block in `.vigie.yaml` so CI and local runs stay consistent.
 
 ```yaml
 suite: <name>            # human-readable suite name
+tier: template           # highest tier this suite's matchers may need (optional)
 templates:               # limit rendering to these templates (optional)
   - templates/deployment.yaml
 
@@ -248,6 +252,42 @@ Map keys that contain dots or slashes (such as the `app.kubernetes.io/name` labe
 bracket segment: `metadata.labels["app.kubernetes.io/name"]` (single or double quotes). The same
 keys are also reachable from `expr:` via CEL, e.g. `doc.metadata.labels["app.kubernetes.io/name"]`.
 
+### Tiers, and the `tier:` field
+
+Matchers need different things from the environment. `equal:` reads a rendered manifest;
+`applies:` needs an API server to admit it; `logsContain:` needs a pod actually running. Those
+requirements form a ladder, and each rung provides everything below it:
+
+| `tier:` | provides | run it with |
+|---|---|---|
+| `template` (default) | rendered manifests | `vigie test` |
+| `apiserver` | + API-server admission, live reads | `vigie test --cluster envtest` |
+| `e2e` | + controllers, running pods, logs, network | `vigie test --cluster kind\|k3d\|kubeconfig` |
+
+A suite declares the highest tier its matchers may require:
+
+```yaml
+suite: app deploys and becomes ready
+tier: e2e
+```
+
+`tier:` is a **ceiling on what the file may contain, not a floor on where it runs.** Use a
+matcher above the declared tier and the file fails to load, naming the matcher and the fix:
+
+```
+test "logs show boot" uses "logsContain", which needs runningPods + podLogs and so
+cannot run at tier template: declare `tier: e2e` on the suite, or use a matcher that
+works at template
+```
+
+Omit it and the suite is held to `template` — so a render-only suite needs no ceremony, and a
+file that quietly grows a cluster matcher is caught rather than silently skipped later.
+
+Running *below* a suite's tier is fine: matchers that need more are reported as skips naming
+the flag that would run them, and a run where everything skipped fails rather than reporting a
+green pass over nothing. Running *above* it is fine too — a `tier: template` suite runs
+unchanged under `--cluster kind`, which is why the same files serve the fast loop and CI.
+
 ### Selecting a document
 
 When a test renders multiple documents, pin one with `target:`:
@@ -293,21 +333,33 @@ tests:
 
 ### Editor autocomplete
 
-`vigie schema` prints the test-file JSON Schema. Reference it from a test file with a
+`vigie schema` prints the test-file JSON Schema, `vigie schema config` the one for
+`.vigie.yaml`. Reference either from the matching file with a
 [yaml-language-server](https://github.com/redhat-developer/yaml-language-server) modeline for
 completion and validation as you type — either the hosted schema:
 
 ```yaml
+# in tests/**/*_test.yaml
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/testfile.json
+
+# in .vigie.yaml
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/config.json
 ```
 
 or a local copy for offline/pinned use:
 
 ```sh
 vigie schema > .vigie.schema.json
 # then:  # yaml-language-server: $schema=./.vigie.schema.json
+
+vigie schema config > .vigie.config.schema.json
+# then:  # yaml-language-server: $schema=./.vigie.config.schema.json
 ```
 
+Both schemas are generated from the Go types they describe (`internal/dsl` and
+`internal/config`), and `.vigie.yaml` is validated against its schema at load time — so a
+mistyped key names itself instead of being silently ignored.
+
 ---
 
 ## CLI reference
@@ -344,7 +396,7 @@ vigie validate [chart]        chart tier: render values.yaml + overlays, validat
   --set / --set-json / --set-literal <k=v>   value overrides (helm semantics)
   -p, --parallelism <n>       parallel scenarios (default: CPU count)
 
-vigie schema                  print the test-file JSON Schema
+vigie schema [target]         print a JSON Schema: testfile (default) or config
 ```
 
 Chart commands default `[chart]` to the current directory, so `vigie test` works from inside a
@@ -381,17 +433,32 @@ validate:
       messageRegex: "networking.k8s.io/v1"
 
 test:
-  testsDir: tests/unit
+  testsDir: tests            # single root holding every test file, scanned recursively
   skipSchema: false          # kubeconform runs per test by default; true opts out
   kubeVersions: [1.36.1]     # kubeconform runs once per version (matrix)
 
-testApply:                   # the apply tier of `vigie test --cluster <backend>`
+  # Per-backend settings for the cluster tiers. These do not select a tier —
+  # `--cluster <backend>` does, and only that backend's block is read.
   cluster:
-    type: envtest            # envtest|kubeconfig|kind|k3d
-    kubeVersion: 1.36.1
-    extraArgs: []            # kind/k3d only, e.g. ["--config", "kind-3node.yaml"]
+    envtest:
+      kubeVersion: 1.36.1    # envtest binary assets (apiserver, etcd)
+    kind:
+      kubeVersion: 1.36.1    # node image
+      binary: ""             # kind CLI; empty = PATH, then cache, then download
+      extraArgs: []          # e.g. ["--config", "kind-3node.yaml"]
+    k3d:
+      kubeVersion: 1.36.1
+      binary: ""
+      extraArgs: []          # e.g. ["-v", "/host:/node"]
+    kubeconfig:
+      path: /home/me/.kube/config   # no `~` expansion; required for --cluster kubeconfig
 ```
 
+CLI flags win over `.vigie.yaml`: `--kube-version`, `--kubeconfig`, `--kind-binary`, and
+`--k3d-binary` each override the selected backend's block. `--download-tools` has no config
+counterpart on purpose — whether a missing CLI may be fetched is an environment concern (TTY vs
+CI), not a per-chart one.
+
 ### Lint rule sets
 
 | Rule set | Checks |

cmd/vigie/outcomes_test.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/fregateops/vigie/internal/runner"
+)
+
+func suiteWith(results ...runner.TestResult) runner.SuiteResult {
+	return runner.SuiteResult{Results: results}
+}
+
+func TestCountTestOutcomes_SplitsSkippedFromExecuted(t *testing.T) {
+	// Skipped tests carry Pass: true, so counting "cases" alone cannot tell a
+	// green run from one that verified nothing.
+	cases := []struct {
+		name         string
+		results      []runner.SuiteResult
+		wantExecuted int
+		wantSkipped  int
+	}{
+		{
+			name:         "no suites",
+			results:      nil,
+			wantExecuted: 0,
+			wantSkipped:  0,
+		},
+		{
+			name:         "files parsed but no tests",
+			results:      []runner.SuiteResult{suiteWith()},
+			wantExecuted: 0,
+			wantSkipped:  0,
+		},
+		{
+			name: "every test skipped",
+			results: []runner.SuiteResult{suiteWith(
+				runner.TestResult{Pass: true, Skipped: true},
+				runner.TestResult{Pass: true, Skipped: true},
+			)},
+			wantExecuted: 0,
+			wantSkipped:  2,
+		},
+		{
+			name: "mixed across suites",
+			results: []runner.SuiteResult{
+				suiteWith(runner.TestResult{Pass: true}, runner.TestResult{Pass: true, Skipped: true}),
+				suiteWith(runner.TestResult{Pass: false}),
+			},
+			wantExecuted: 2,
+			wantSkipped:  1,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			executed, skipped := countTestOutcomes(tc.results)
+			if executed != tc.wantExecuted || skipped != tc.wantSkipped {
+				t.Errorf("countTestOutcomes = (%d executed, %d skipped), want (%d, %d)",
+					executed, skipped, tc.wantExecuted, tc.wantSkipped)
+			}
+		})
+	}
+}

cmd/vigie/schema.go

@@ -4,22 +4,67 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/fregateops/vigie/internal/config"
 	"github.com/fregateops/vigie/internal/dsl"
 	"github.com/spf13/cobra"
 )
 
+// Schema targets accepted by `vigie schema`. testfile stays the default so the
+// documented `vigie schema > .vigie.schema.json` idiom keeps working.
+const (
+	schemaTargetTestFile = "testfile"
+	schemaTargetConfig   = "config"
+)
+
 var schemaCmd = &cobra.Command{
-	Use:   "schema",
-	Short: "Print the test file JSON Schema",
-	Example: `  # Save the schema for editor autocomplete, then reference it from a
-  # test file with:  # yaml-language-server: $schema=./.vigie.schema.json
-  vigie schema > .vigie.schema.json`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Fprintf(os.Stdout, "%s\n", dsl.SchemaJSON())
-		return nil
-	},
+	Use:   "schema [testfile|config]",
+	Short: "Print a JSON Schema: the test file format (default) or .vigie.yaml",
+	Long: "Print one of vigie's JSON Schemas, for editor autocomplete and validation:\n\n" +
+		"  testfile  the test file format (tests/**/*_test.yaml) — the default\n" +
+		"  config    the per-chart configuration file (.vigie.yaml)",
+	Example: `  # Save the test file schema, then reference it from a test file with:
+  #   # yaml-language-server: $schema=./.vigie.schema.json
+  vigie schema > .vigie.schema.json
+
+  # Save the config schema, then reference it from .vigie.yaml with:
+  #   # yaml-language-server: $schema=./.vigie.config.schema.json
+  vigie schema config > .vigie.config.schema.json`,
+	Args:      cobra.MaximumNArgs(1),
+	ValidArgs: []string{schemaTargetTestFile, schemaTargetConfig},
+	RunE:      runSchemaCmd,
 }
 
 func init() {
 	rootCmd.AddCommand(schemaCmd)
 }
+
+func runSchemaCmd(cmd *cobra.Command, args []string) error {
+	schema, err := schemaFor(schemaTarget(args))
+	if err != nil {
+		exitErr(3, "%v", err)
+	}
+	fmt.Fprintf(os.Stdout, "%s\n", schema)
+	return nil
+}
+
+// schemaTarget returns the requested target, defaulting to the test file so a
+// bare `vigie schema` keeps printing it.
+func schemaTarget(args []string) string {
+	if len(args) == 1 {
+		return args[0]
+	}
+	return schemaTargetTestFile
+}
+
+// schemaFor returns the embedded schema a target name selects.
+func schemaFor(target string) ([]byte, error) {
+	switch target {
+	case schemaTargetTestFile:
+		return dsl.SchemaJSON(), nil
+	case schemaTargetConfig:
+		return config.SchemaJSON(), nil
+	default:
+		return nil, fmt.Errorf("unknown schema %q: valid values are %s, %s",
+			target, schemaTargetTestFile, schemaTargetConfig)
+	}
+}

cmd/vigie/schema_test.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+func TestSchemaFor_SelectsTheRightDocument(t *testing.T) {
+	cases := []struct {
+		target   string
+		idSuffix string
+	}{
+		{schemaTargetTestFile, "/testfile.json"},
+		{schemaTargetConfig, "/config.json"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.target, func(t *testing.T) {
+			schema, err := schemaFor(tc.target)
+			if err != nil {
+				t.Fatalf("schemaFor(%q): %v", tc.target, err)
+			}
+			if !bytes.Contains(schema, []byte(tc.idSuffix+`"`)) {
+				t.Errorf("schema for %q does not carry an $id ending in %q", tc.target, tc.idSuffix)
+			}
+		})
+	}
+}
+
+func TestSchemaFor_UnknownTargetListsValidOnes(t *testing.T) {
+	_, err := schemaFor("values")
+	if err == nil {
+		t.Fatal("schemaFor must reject an unknown target, got nil error")
+	}
+	for _, want := range []string{"values", schemaTargetTestFile, schemaTargetConfig} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error %q does not mention %q", err, want)
+		}
+	}
+}
+
+// A bare `vigie schema` must keep printing the test-file schema: the README
+// documents `vigie schema > .vigie.schema.json` as the way to get it.
+func TestSchemaTarget_DefaultsToTheTestFile(t *testing.T) {
+	if got, want := schemaTarget(nil), schemaTargetTestFile; got != want {
+		t.Errorf("schemaTarget(nil): want %q, got %q", want, got)
+	}
+	if got, want := schemaTarget([]string{schemaTargetConfig}), schemaTargetConfig; got != want {
+		t.Errorf("schemaTarget([config]): want %q, got %q", want, got)
+	}
+}