From 2f914204499957b1068f95a9d43f14b8a283ca2c Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 30 Jun 2026 15:58:55 +0200 Subject: [PATCH 1/2] Initial code version. --- Makefile | 10 +- apps/cli/pyproject.toml | 1 + .../commands/coverage_matrix.py | 76 +++++ apps/cli/src/living_doc_cli/main.py | 2 + apps/cli/tests/test_cli.py | 111 +++++++ packages/services/coverage_matrix/README.md | 57 ++++ .../services/coverage_matrix/pyproject.toml | 54 ++++ .../__init__.py | 10 + .../loader.py | 57 ++++ .../matcher.py | 133 +++++++++ .../model/__init__.py | 3 + .../model/coverage_item.py | 112 ++++++++ .../schema/coverage-matrix-v1.0.0-schema.json | 270 ++++++++++++++++++ .../service.py | 94 ++++++ .../summary.py | 58 ++++ .../coverage_matrix/tests/__init__.py | 3 + .../tests/fixtures/golden/doc_source.json | 90 ++++++ .../golden/expected_coverage_matrix.json | 213 ++++++++++++++ .../tests/fixtures/golden/ui_tests.json | 146 ++++++++++ .../tests/integration/__init__.py | 3 + .../tests/integration/test_golden_files.py | 76 +++++ .../coverage_matrix/tests/test_loader.py | 76 +++++ .../coverage_matrix/tests/test_matcher.py | 160 +++++++++++ .../coverage_matrix/tests/test_service.py | 106 +++++++ .../coverage_matrix/tests/test_summary.py | 71 +++++ 25 files changed, 1990 insertions(+), 2 deletions(-) create mode 100644 apps/cli/src/living_doc_cli/commands/coverage_matrix.py create mode 100644 packages/services/coverage_matrix/README.md create mode 100644 packages/services/coverage_matrix/pyproject.toml create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/__init__.py create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/loader.py create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/matcher.py create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/__init__.py create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/coverage_item.py create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/schema/coverage-matrix-v1.0.0-schema.json create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/service.py create mode 100644 packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/summary.py create mode 100644 packages/services/coverage_matrix/tests/__init__.py create mode 100644 packages/services/coverage_matrix/tests/fixtures/golden/doc_source.json create mode 100644 packages/services/coverage_matrix/tests/fixtures/golden/expected_coverage_matrix.json create mode 100644 packages/services/coverage_matrix/tests/fixtures/golden/ui_tests.json create mode 100644 packages/services/coverage_matrix/tests/integration/__init__.py create mode 100644 packages/services/coverage_matrix/tests/integration/test_golden_files.py create mode 100644 packages/services/coverage_matrix/tests/test_loader.py create mode 100644 packages/services/coverage_matrix/tests/test_matcher.py create mode 100644 packages/services/coverage_matrix/tests/test_service.py create mode 100644 packages/services/coverage_matrix/tests/test_summary.py diff --git a/Makefile b/Makefile index 7d081c5..758ebe2 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ PACKAGES := packages/core \ packages/datasets_pdf \ packages/adapters/collector_gh \ - packages/services/normalize_issues + packages/services/normalize_issues \ + packages/services/coverage_matrix APPS := apps/cli ALL_TARGETS := $(PACKAGES) $(APPS) @@ -30,6 +31,7 @@ install: ## Install all packages with [dev] dependencies $(PYTHON) -m pip install -e packages/datasets_pdf[dev] $(PYTHON) -m pip install -e packages/adapters/collector_gh[dev] $(PYTHON) -m pip install -e packages/services/normalize_issues[dev] + $(PYTHON) -m pip install -e packages/services/coverage_matrix[dev] $(PYTHON) -m pip install -e "apps/cli[dev]" @echo "$(GREEN)✓ All packages installed$(NC)" @@ -53,6 +55,7 @@ help: ## Show this help message @echo " make py-qa-datasets-pdf Run all QA on packages/datasets_pdf" @echo " make py-qa-collector-gh Run all QA on packages/adapters/collector_gh" @echo " make py-qa-normalize Run all QA on packages/services/normalize_issues" + @echo " make py-qa-coverage Run all QA on packages/services/coverage_matrix" @echo " make py-qa-cli Run all QA on apps/cli" @echo "" @echo "$(YELLOW)Run individual checks by package:$(NC)" @@ -62,7 +65,7 @@ help: ## Show this help message @echo " make pytest-unit-core Run tests on packages/core" @echo "" @echo "$(YELLOW)Package shortcuts (replace 'core' with any package):$(NC)" - @echo " - datasets_pdf, collector_gh, normalize, cli" + @echo " - datasets_pdf, collector_gh, normalize, coverage, cli" # ============================================================================ # ALL PACKAGES: Aggregated QA targets @@ -111,6 +114,9 @@ py-qa-collector-gh: black-packages/adapters/collector_gh pylint-packages/adapter py-qa-normalize: black-packages/services/normalize_issues pylint-packages/services/normalize_issues mypy-packages/services/normalize_issues pytest-unit-packages/services/normalize_issues @echo "$(GREEN)✓ All QA checks passed for packages/services/normalize_issues$(NC)" +py-qa-coverage: black-packages/services/coverage_matrix pylint-packages/services/coverage_matrix mypy-packages/services/coverage_matrix pytest-unit-packages/services/coverage_matrix + @echo "$(GREEN)✓ All QA checks passed for packages/services/coverage_matrix$(NC)" + py-qa-cli: black-apps/cli pylint-apps/cli mypy-apps/cli pytest-unit-apps/cli @echo "$(GREEN)✓ All QA checks passed for apps/cli$(NC)" diff --git a/apps/cli/pyproject.toml b/apps/cli/pyproject.toml index 4833794..1ae5c5d 100644 --- a/apps/cli/pyproject.toml +++ b/apps/cli/pyproject.toml @@ -15,6 +15,7 @@ requires-python = ">=3.14" dependencies = [ "living-doc-core", "living-doc-service-normalize-issues", + "living-doc-service-coverage-matrix", "click>=8.1.0,<9.0.0" ] diff --git a/apps/cli/src/living_doc_cli/commands/coverage_matrix.py b/apps/cli/src/living_doc_cli/commands/coverage_matrix.py new file mode 100644 index 0000000..f0d9bef --- /dev/null +++ b/apps/cli/src/living_doc_cli/commands/coverage_matrix.py @@ -0,0 +1,76 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +""" +coverage-matrix CLI command. +""" + +import sys + +import click + +from living_doc_core.errors import ToolkitError # type: ignore[import-untyped] +from living_doc_service_coverage_matrix.service import run_service # type: ignore[import-untyped] + + +@click.command("coverage-matrix") +@click.option( + "--doc-input", + "doc_input", + required=True, + type=click.Path(exists=False), + help="Path to the US+AC doc JSON file (doc-source.json / doc-issues.json)", +) +@click.option( + "--tests-input", + "tests_input", + required=True, + type=click.Path(exists=False), + help="Path to the ui-tests JSON file", +) +@click.option( + "--output", + "output_path", + required=True, + type=click.Path(), + help="Destination path for coverage-matrix.json", +) +@click.option( + "--fail-under", + "fail_under", + type=float, + default=None, + help="Exit with code 1 if coverage_pct is below this threshold", +) +@click.option("--verbose", is_flag=True, help="Enable verbose logging") +@click.pass_context +def coverage_matrix( # pylint: disable=too-many-arguments,too-many-positional-arguments + ctx: click.Context, + doc_input: str, + tests_input: str, + output_path: str, + fail_under: float | None, + verbose: bool, +) -> None: + """ + Generate an AC-level test coverage matrix. + + Cross-references doc-source output (User Stories + acceptance criteria) with + ui-tests output (test scenarios) into coverage-matrix.json. + """ + global_verbose = ctx.obj.get("verbose", False) if ctx.obj else False + options = { + "verbose": verbose or global_verbose, + "fail_under": fail_under, + } + + try: + run_service(doc_input, tests_input, output_path, options) + click.echo(f"Successfully generated coverage matrix -> {output_path}") + + except ToolkitError as e: + click.echo(f"Error: {e.message}", err=True) + sys.exit(1) + + except Exception as e: # pylint: disable=broad-exception-caught + click.echo(f"Error: Unexpected error: {e}", err=True) + sys.exit(1) diff --git a/apps/cli/src/living_doc_cli/main.py b/apps/cli/src/living_doc_cli/main.py index 1ba05ab..d64c7ac 100644 --- a/apps/cli/src/living_doc_cli/main.py +++ b/apps/cli/src/living_doc_cli/main.py @@ -7,6 +7,7 @@ import click from living_doc_cli import __version__ +from living_doc_cli.commands.coverage_matrix import coverage_matrix from living_doc_cli.commands.normalize_issues import normalize_issues @@ -27,6 +28,7 @@ def cli(ctx: click.Context, verbose: bool) -> None: cli.add_command(normalize_issues) +cli.add_command(coverage_matrix) if __name__ == "__main__": diff --git a/apps/cli/tests/test_cli.py b/apps/cli/tests/test_cli.py index 7ac41f9..aa0d204 100644 --- a/apps/cli/tests/test_cli.py +++ b/apps/cli/tests/test_cli.py @@ -221,3 +221,114 @@ def test_normalize_issues_unexpected_error(mock_run_service, runner): assert result.exit_code == 1 assert "Unexpected error:" in result.output assert "Unexpected failure" in result.output + + +def test_cli_help_lists_coverage_matrix(runner): + """Test that coverage-matrix appears in the top-level help.""" + result = runner.invoke(cli, ["--help"]) + assert result.exit_code == 0 + assert "coverage-matrix" in result.output + + +def test_coverage_matrix_help(runner): + """Test that coverage-matrix --help displays usage information.""" + result = runner.invoke(cli, ["coverage-matrix", "--help"]) + assert result.exit_code == 0 + assert "--doc-input" in result.output + assert "--tests-input" in result.output + assert "--output" in result.output + assert "--fail-under" in result.output + + +def test_coverage_matrix_missing_required_args(runner): + """Test that missing required arguments shows an error.""" + result = runner.invoke(cli, ["coverage-matrix"]) + assert result.exit_code != 0 + assert "Missing option" in result.output or "Error" in result.output + + +@patch("living_doc_cli.commands.coverage_matrix.run_service") +def test_coverage_matrix_success(mock_run_service, runner): + """Test successful execution of coverage-matrix command.""" + mock_run_service.return_value = None + + result = runner.invoke( + cli, + [ + "coverage-matrix", + "--doc-input", + "doc.json", + "--tests-input", + "tests.json", + "--output", + "out.json", + ], + ) + + assert result.exit_code == 0 + assert "Successfully generated coverage matrix" in result.output + mock_run_service.assert_called_once() + + call_args = mock_run_service.call_args + assert call_args[0][0] == "doc.json" + assert call_args[0][1] == "tests.json" + assert call_args[0][2] == "out.json" + options = call_args[0][3] + assert options["verbose"] is False + assert options["fail_under"] is None + + +@patch("living_doc_cli.commands.coverage_matrix.run_service") +def test_coverage_matrix_with_fail_under_and_verbose(mock_run_service, runner): + """Test coverage-matrix with --fail-under and --verbose options.""" + mock_run_service.return_value = None + + result = runner.invoke( + cli, + [ + "coverage-matrix", + "--doc-input", + "doc.json", + "--tests-input", + "tests.json", + "--output", + "out.json", + "--fail-under", + "75", + "--verbose", + ], + ) + + assert result.exit_code == 0 + options = mock_run_service.call_args[0][3] + assert options["verbose"] is True + assert options["fail_under"] == 75.0 + + +@patch("living_doc_cli.commands.coverage_matrix.run_service") +def test_coverage_matrix_toolkit_error_exits_1(mock_run_service, runner): + """Test exit code 1 when the service raises a ToolkitError.""" + mock_run_service.side_effect = InvalidInputError("Doc input must contain an 'items' array") + + result = runner.invoke( + cli, + ["coverage-matrix", "--doc-input", "doc.json", "--tests-input", "tests.json", "--output", "out.json"], + ) + + assert result.exit_code == 1 + assert "Doc input must contain an 'items' array" in result.output + + +@patch("living_doc_cli.commands.coverage_matrix.run_service") +def test_coverage_matrix_unexpected_error_exits_1(mock_run_service, runner): + """Test exit code 1 for unexpected errors.""" + mock_run_service.side_effect = RuntimeError("boom") + + result = runner.invoke( + cli, + ["coverage-matrix", "--doc-input", "doc.json", "--tests-input", "tests.json", "--output", "out.json"], + ) + + assert result.exit_code == 1 + assert "Unexpected error:" in result.output + assert "boom" in result.output diff --git a/packages/services/coverage_matrix/README.md b/packages/services/coverage_matrix/README.md new file mode 100644 index 0000000..05a1d15 --- /dev/null +++ b/packages/services/coverage_matrix/README.md @@ -0,0 +1,57 @@ +# living-doc-service-coverage-matrix + +Coverage matrix generator for the Living Documentation Toolkit. + +Cross-references two collector outputs to produce an AC-level test coverage matrix +per User Story, ready for PDF report generation. + +## Inputs + +| Input | Source | Description | +|---|---|---| +| `doc-source.json` | living-doc-collector-gh `doc-source` mode | User Stories with acceptance criteria | +| `ui-tests.json` | living-doc-collector-gh `ui-tests` mode | UI/E2E test scenarios with `us_id` / `ac_ids` | + +The doc input may be a bare JSON array of User Story objects or a collector envelope +exposing an `items` array. The tests input must be an envelope with an `items` array. + +## Output + +`coverage-matrix.json`, conforming to +[`coverage-matrix-v1.0.0-schema.json`](src/living_doc_service_coverage_matrix/schema/coverage-matrix-v1.0.0-schema.json). + +## CLI + +``` +living-doc coverage-matrix \ + --doc-input doc-source.json \ + --tests-input ui-tests.json \ + --output coverage-matrix.json \ + [--fail-under 80] +``` + +| Argument | Required | Description | +|---|---|---| +| `--doc-input` | yes | Path to the US+AC doc JSON file | +| `--tests-input` | yes | Path to the ui-tests JSON file | +| `--output` | yes | Destination path for the coverage matrix JSON | +| `--fail-under` | no | Exit with code 1 if `coverage_pct < N` | + +## Matching logic + +- A scenario links to a User Story when `scenario.us_id` equals the US short id + (the numeric suffix of `us.id`, e.g. `org/repo/US-27` -> `US-27`). +- A scenario covers an AC when the `ac_id` appears in `scenario.ac_ids` and matches a + known `acceptance_criteria[].id` on the resolved User Story. +- `coverage_pct` counts only Active ACs, so deprecated ACs never inflate it. +- Scenarios with an unresolved `us_id` land in `unlinked_tests`; `ac_ids` that do not + exist on the resolved US land in `stale_ac_refs`. + +## Module layout + +- `loader.py` — pure I/O: `load_doc_input()`, `load_tests_input()` +- `matcher.py` — pure: `build_coverage_matrix()` +- `summary.py` — pure: `compute_summary()`, `compute_us_summary()` +- `service.py` — orchestration: `run_service()` +- `model/coverage_item.py` — output dataclasses +- `schema/` — shipped JSON Schema diff --git a/packages/services/coverage_matrix/pyproject.toml b/packages/services/coverage_matrix/pyproject.toml new file mode 100644 index 0000000..fe85994 --- /dev/null +++ b/packages/services/coverage_matrix/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "living-doc-service-coverage-matrix" +version = "1.0.0" +description = "Coverage matrix generator for the Living Documentation Toolkit" +readme = "README.md" +license = {text = "Apache-2.0"} +authors = [ + {name = "AbsaOSS", email = "opensource@absa.africa"} +] +requires-python = ">=3.14" +dependencies = [ + "living-doc-core" +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "pylint>=3.0.0", + "mypy>=1.0.0" +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +"living_doc_service_coverage_matrix" = ["schema/*.json"] + +[tool.black] +line-length = 120 +target-version = ['py314'] +extend-exclude = ''' +^( + tests/ + | verifications/ +) +''' + +[tool.pylint.format] +max-line-length = 120 + +[tool.mypy] +check_untyped_defs = true +python_version = "3.14" +exclude = '^(tests/|verifications/)' + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--cov=src --cov-report=term --cov-report=html --cov-fail-under=80" diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/__init__.py b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/__init__.py new file mode 100644 index 0000000..d824fd4 --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/__init__.py @@ -0,0 +1,10 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +""" +Living Documentation Toolkit - Coverage Matrix Service. + +This service cross-references collector doc-source output with ui-tests output to +produce an AC-level test coverage matrix per User Story. +""" + +__version__ = "1.0.0" diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/loader.py b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/loader.py new file mode 100644 index 0000000..197bd7c --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/loader.py @@ -0,0 +1,57 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +""" +Input loaders for the coverage-matrix service. + +Pure I/O boundary: read JSON files and extract the item arrays. No matching logic. +""" + +from pathlib import Path + +from living_doc_core.errors import InvalidInputError # type: ignore[import-untyped] +from living_doc_core.json_utils import read_json # type: ignore[import-untyped] + + +def load_doc_input(filepath: str | Path) -> list[dict]: + """ + Load the User Story + AC doc input. + + Accepts either a bare JSON array of User Story objects or a collector envelope + object exposing an ``items`` array. + + Args: + filepath: Path to the doc JSON file (doc-source.json / doc-issues.json) + + Returns: + List of User Story dictionaries + + Raises: + FileIOError: If the file is missing or unreadable + InvalidInputError: If the JSON is malformed or not a US array/envelope + """ + payload = read_json(filepath) + if isinstance(payload, list): + return payload + if isinstance(payload, dict) and isinstance(payload.get("items"), list): + return payload["items"] + raise InvalidInputError(f"Doc input '{filepath}' must be a JSON array or an object with an 'items' array") + + +def load_tests_input(filepath: str | Path) -> list[dict]: + """ + Load the ui-tests input. + + Args: + filepath: Path to the ui-tests JSON file + + Returns: + List of test scenario dictionaries + + Raises: + FileIOError: If the file is missing or unreadable + InvalidInputError: If the JSON is malformed or has no ``items`` array + """ + payload = read_json(filepath) + if isinstance(payload, dict) and isinstance(payload.get("items"), list): + return payload["items"] + raise InvalidInputError(f"Tests input '{filepath}' must contain an 'items' array") diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/matcher.py b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/matcher.py new file mode 100644 index 0000000..8c68526 --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/matcher.py @@ -0,0 +1,133 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +""" +Coverage matcher. + +Pure transformation: takes parsed doc and tests item lists and returns a +:class:`CoverageMatrix`. No file I/O and no logging — fully unit-testable. +""" + +from collections import defaultdict + +from living_doc_service_coverage_matrix.model.coverage_item import ( + AcCoverage, + Coverage, + CoverageMatrix, + StaleAcRef, + TestRef, + UnlinkedTest, + UserStoryCoverage, +) +from living_doc_service_coverage_matrix.summary import compute_summary, compute_us_summary + +SCHEMA_VERSION = "coverage-matrix-v1.0.0" +COVERED = "covered" +NOT_COVERED = "not_covered" + + +def _us_num(full_id: str) -> str: + """Extract the short US id (e.g. ``US-27``) from a full id like ``org/repo/US-27``.""" + return full_id.split("/")[-1] + + +def _test_ref(scenario: dict) -> TestRef: + """Build a TestRef from a scenario dictionary.""" + return TestRef( + id=scenario.get("id"), + scenario_name=scenario.get("scenario_name"), + tags=scenario.get("tags") or [], + source=scenario.get("source"), + ) + + +def _resolve_tests( + test_items: list[dict], + us_by_num: dict[str, dict], +) -> tuple[dict[str, dict[str, list[TestRef]]], list[UnlinkedTest], list[StaleAcRef]]: + """Split scenarios into a per-US coverage map, unlinked tests, and stale AC refs.""" + coverage_map: dict[str, dict[str, list[TestRef]]] = defaultdict(lambda: defaultdict(list)) + unlinked: list[UnlinkedTest] = [] + stale: list[StaleAcRef] = [] + + for scenario in test_items: + us_id = scenario.get("us_id") + if not us_id or us_id not in us_by_num: + unlinked.append( + UnlinkedTest( + id=scenario.get("id"), + scenario_name=scenario.get("scenario_name"), + us_id=us_id, + ac_ids=scenario.get("ac_ids") or [], + source=scenario.get("source"), + ) + ) + continue + + valid_ac_ids = {ac.get("id") for ac in us_by_num[us_id]["acceptance_criteria"]} + for ac_id in scenario.get("ac_ids") or []: + if ac_id in valid_ac_ids: + coverage_map[us_id][ac_id].append(_test_ref(scenario)) + else: + stale.append( + StaleAcRef( + scenario_id=scenario.get("id"), + scenario_name=scenario.get("scenario_name"), + us_id=us_id, + stale_ac_id=ac_id, + source=scenario.get("source"), + ) + ) + + return coverage_map, unlinked, stale + + +def _build_user_story(us: dict, ac_tests: dict[str, list[TestRef]]) -> UserStoryCoverage: + """Build a single UserStoryCoverage from a US dict and its resolved AC tests.""" + ac_coverages: list[AcCoverage] = [] + for ac in us["acceptance_criteria"]: + tests = ac_tests.get(ac.get("id"), []) + status = COVERED if tests else NOT_COVERED + ac_coverages.append( + AcCoverage( + id=ac.get("id"), + state=ac.get("state"), + version=ac.get("version"), + description=ac.get("description"), + coverage=Coverage(status=status, test_count=len(tests), tests=tests), + ) + ) + return UserStoryCoverage( + id=_us_num(us["id"]), + full_id=us["id"], + title=us.get("title"), + state=us.get("state"), + summary=compute_us_summary(ac_coverages), + acceptance_criteria=ac_coverages, + ) + + +def build_coverage_matrix(doc_items: list[dict], test_items: list[dict], generated_at: str) -> CoverageMatrix: + """ + Cross-reference User Stories and UI tests into a coverage matrix. + + Args: + doc_items: User Story dictionaries (each with ``id`` and ``acceptance_criteria``) + test_items: UI test scenario dictionaries + generated_at: ISO-8601 timestamp recorded on the matrix + + Returns: + A fully populated :class:`CoverageMatrix`. + """ + us_by_num = {_us_num(us["id"]): us for us in doc_items} + coverage_map, unlinked, stale = _resolve_tests(test_items, us_by_num) + + user_stories = [_build_user_story(us, coverage_map.get(_us_num(us["id"]), {})) for us in doc_items] + + return CoverageMatrix( + schema_version=SCHEMA_VERSION, + generated_at=generated_at, + summary=compute_summary(user_stories), + user_stories=user_stories, + unlinked_tests=unlinked, + stale_ac_refs=stale, + ) diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/__init__.py b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/__init__.py new file mode 100644 index 0000000..5f860b8 --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Dataclasses describing the coverage-matrix output schema.""" diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/coverage_item.py b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/coverage_item.py new file mode 100644 index 0000000..6f727a8 --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/coverage_item.py @@ -0,0 +1,112 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +""" +Dataclasses for the coverage-matrix output document. + +These types mirror ``coverage-matrix-v1.0.0-schema.json`` and serialize to plain +dictionaries via :meth:`CoverageMatrix.to_dict`. +""" + +from dataclasses import asdict, dataclass +from typing import Any, Optional + + +@dataclass +class Summary: + """AC coverage tallies scoped to a single User Story.""" + + total_acs: int + active_acs: int + covered_acs: int + coverage_pct: Optional[float] + + +@dataclass +class TopSummary: + """AC coverage tallies aggregated across all User Stories.""" + + total_user_stories: int + total_acs: int + active_acs: int + covered_acs: int + coverage_pct: Optional[float] + + +@dataclass +class TestRef: + """A reference to a UI test scenario covering an acceptance criterion.""" + + id: Optional[str] + scenario_name: Optional[str] + tags: list[str] + source: Optional[dict[str, Any]] + + +@dataclass +class Coverage: + """Coverage status and linked tests for a single acceptance criterion.""" + + status: str + test_count: int + tests: list[TestRef] + + +@dataclass +class AcCoverage: + """An acceptance criterion with its resolved coverage.""" + + id: Optional[str] + state: Optional[str] + version: Optional[str] + description: Optional[str] + coverage: Coverage + + +@dataclass +class UserStoryCoverage: + """A User Story with per-AC coverage and a scoped summary.""" + + id: str + full_id: str + title: Optional[str] + state: Optional[str] + summary: Summary + acceptance_criteria: list[AcCoverage] + + +@dataclass +class UnlinkedTest: + """A scenario whose ``us_id`` is null or does not resolve to a known US.""" + + id: Optional[str] + scenario_name: Optional[str] + us_id: Optional[str] + ac_ids: list[str] + source: Optional[dict[str, Any]] + + +@dataclass +class StaleAcRef: + """An ``ac_id`` referenced by a scenario that does not exist on the resolved US.""" + + scenario_id: Optional[str] + scenario_name: Optional[str] + us_id: Optional[str] + stale_ac_id: str + source: Optional[dict[str, Any]] + + +@dataclass +class CoverageMatrix: + """Top-level coverage-matrix document.""" + + schema_version: str + generated_at: str + summary: TopSummary + user_stories: list[UserStoryCoverage] + unlinked_tests: list[UnlinkedTest] + stale_ac_refs: list[StaleAcRef] + + def to_dict(self) -> dict[str, Any]: + """Serialize the matrix to a plain dictionary suitable for JSON output.""" + return asdict(self) diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/schema/coverage-matrix-v1.0.0-schema.json b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/schema/coverage-matrix-v1.0.0-schema.json new file mode 100644 index 0000000..39ea44c --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/schema/coverage-matrix-v1.0.0-schema.json @@ -0,0 +1,270 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://absaoss.github.io/living-doc-toolkit/schemas/coverage-matrix-v1.0.0-schema.json", + "title": "Coverage Matrix", + "description": "AC-level test coverage matrix per User Story, produced by the coverage-matrix generator.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "generated_at", + "summary", + "user_stories", + "unlinked_tests", + "stale_ac_refs" + ], + "properties": { + "schema_version": { + "type": "string", + "const": "coverage-matrix-v1.0.0" + }, + "generated_at": { + "type": "string", + "description": "ISO-8601 timestamp of generation." + }, + "summary": { + "$ref": "#/$defs/topSummary" + }, + "user_stories": { + "type": "array", + "items": { + "$ref": "#/$defs/userStory" + } + }, + "unlinked_tests": { + "type": "array", + "items": { + "$ref": "#/$defs/unlinkedTest" + } + }, + "stale_ac_refs": { + "type": "array", + "items": { + "$ref": "#/$defs/staleAcRef" + } + } + }, + "$defs": { + "source": { + "type": ["object", "null"], + "properties": { + "org": { + "type": ["string", "null"] + }, + "repo": { + "type": ["string", "null"] + }, + "file": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["total_acs", "active_acs", "covered_acs", "coverage_pct"], + "properties": { + "total_acs": { + "type": "integer", + "minimum": 0 + }, + "active_acs": { + "type": "integer", + "minimum": 0 + }, + "covered_acs": { + "type": "integer", + "minimum": 0 + }, + "coverage_pct": { + "type": ["number", "null"] + } + } + }, + "topSummary": { + "type": "object", + "additionalProperties": false, + "required": [ + "total_user_stories", + "total_acs", + "active_acs", + "covered_acs", + "coverage_pct" + ], + "properties": { + "total_user_stories": { + "type": "integer", + "minimum": 0 + }, + "total_acs": { + "type": "integer", + "minimum": 0 + }, + "active_acs": { + "type": "integer", + "minimum": 0 + }, + "covered_acs": { + "type": "integer", + "minimum": 0 + }, + "coverage_pct": { + "type": ["number", "null"] + } + } + }, + "testRef": { + "type": "object", + "additionalProperties": false, + "required": ["id", "scenario_name", "tags", "source"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "scenario_name": { + "type": ["string", "null"] + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "source": { + "$ref": "#/$defs/source" + } + } + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["status", "test_count", "tests"], + "properties": { + "status": { + "type": "string", + "enum": ["covered", "not_covered"] + }, + "test_count": { + "type": "integer", + "minimum": 0 + }, + "tests": { + "type": "array", + "items": { + "$ref": "#/$defs/testRef" + } + } + } + }, + "acceptanceCriterion": { + "type": "object", + "additionalProperties": false, + "required": ["id", "state", "version", "description", "coverage"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "state": { + "type": ["string", "null"] + }, + "version": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "coverage": { + "$ref": "#/$defs/coverage" + } + } + }, + "userStory": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "full_id", + "title", + "state", + "summary", + "acceptance_criteria" + ], + "properties": { + "id": { + "type": "string" + }, + "full_id": { + "type": "string" + }, + "title": { + "type": ["string", "null"] + }, + "state": { + "type": ["string", "null"] + }, + "summary": { + "$ref": "#/$defs/summary" + }, + "acceptance_criteria": { + "type": "array", + "items": { + "$ref": "#/$defs/acceptanceCriterion" + } + } + } + }, + "unlinkedTest": { + "type": "object", + "additionalProperties": false, + "required": ["id", "scenario_name", "us_id", "ac_ids", "source"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "scenario_name": { + "type": ["string", "null"] + }, + "us_id": { + "type": ["string", "null"] + }, + "ac_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "source": { + "$ref": "#/$defs/source" + } + } + }, + "staleAcRef": { + "type": "object", + "additionalProperties": false, + "required": [ + "scenario_id", + "scenario_name", + "us_id", + "stale_ac_id", + "source" + ], + "properties": { + "scenario_id": { + "type": ["string", "null"] + }, + "scenario_name": { + "type": ["string", "null"] + }, + "us_id": { + "type": ["string", "null"] + }, + "stale_ac_id": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/source" + } + } + } + } +} diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/service.py b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/service.py new file mode 100644 index 0000000..fca1beb --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/service.py @@ -0,0 +1,94 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +""" +Service orchestration for the coverage-matrix generator. + +Loads the doc and tests inputs, builds the coverage matrix, writes the output JSON, +and enforces an optional ``--fail-under`` threshold. +""" + +from datetime import datetime, timezone + +from living_doc_core.errors import ToolkitError # type: ignore[import-untyped] +from living_doc_core.json_utils import write_json # type: ignore[import-untyped] +from living_doc_core.logging_config import setup_logging # type: ignore[import-untyped] + +from living_doc_service_coverage_matrix.loader import load_doc_input, load_tests_input +from living_doc_service_coverage_matrix.matcher import build_coverage_matrix +from living_doc_service_coverage_matrix.model.coverage_item import CoverageMatrix + + +class CoverageThresholdError(ToolkitError): + """Coverage percentage below the configured ``--fail-under`` threshold. Exit code 1.""" + + exit_code = 1 + + +def _filter_valid_user_stories(doc_items: list[dict], logger) -> list[dict]: + """Drop User Stories missing ``id`` or ``acceptance_criteria``, logging each skip.""" + valid: list[dict] = [] + for us in doc_items: + if not isinstance(us, dict) or not us.get("id") or us.get("acceptance_criteria") is None: + skipped_id = us.get("id") if isinstance(us, dict) else us + logger.warning("Skipping user story missing 'id' or 'acceptance_criteria': %s", skipped_id) + continue + valid.append(us) + return valid + + +def run_service(doc_input: str, tests_input: str, output_path: str, options: dict) -> CoverageMatrix: + """ + Run the coverage-matrix generation pipeline. + + Args: + doc_input: Path to the doc JSON file (doc-source.json / doc-issues.json) + tests_input: Path to the ui-tests JSON file + output_path: Destination path for coverage-matrix.json + options: Configuration options (``verbose``, ``fail_under``) + + Returns: + The generated :class:`CoverageMatrix`. + + Raises: + FileIOError: If an input file is missing or the output cannot be written + InvalidInputError: If an input file is malformed or has the wrong shape + CoverageThresholdError: If coverage is below ``--fail-under`` + """ + verbose = options.get("verbose", False) + fail_under = options.get("fail_under") + logger = setup_logging(verbose=verbose) + + logger.info("Starting coverage matrix generation") + logger.info("Doc input: %s", doc_input) + logger.info("Tests input: %s", tests_input) + logger.info("Output: %s", output_path) + + doc_items = load_doc_input(doc_input) + test_items = load_tests_input(tests_input) + + valid_doc = _filter_valid_user_stories(doc_items, logger) + + generated_at = datetime.now(timezone.utc).isoformat() + matrix = build_coverage_matrix(valid_doc, test_items, generated_at) + + logger.info("Writing coverage matrix JSON...") + write_json(output_path, matrix.to_dict(), indent=2, sort_keys=True) + + summary = matrix.summary + logger.info("Coverage matrix written successfully") + logger.info(" - User stories: %d", summary.total_user_stories) + logger.info(" - Total ACs: %d", summary.total_acs) + logger.info(" - Active ACs: %d", summary.active_acs) + logger.info(" - Covered ACs: %d", summary.covered_acs) + logger.info(" - Coverage: %s%%", summary.coverage_pct) + if matrix.unlinked_tests: + logger.warning("Unlinked tests: %d", len(matrix.unlinked_tests)) + if matrix.stale_ac_refs: + logger.warning("Stale AC references: %d", len(matrix.stale_ac_refs)) + + if fail_under is not None: + effective_pct = summary.coverage_pct if summary.coverage_pct is not None else 0.0 + if effective_pct < fail_under: + raise CoverageThresholdError(f"Coverage {effective_pct}% is below threshold {fail_under}%") + + return matrix diff --git a/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/summary.py b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/summary.py new file mode 100644 index 0000000..dafc23e --- /dev/null +++ b/packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/summary.py @@ -0,0 +1,58 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +""" +Summary computation for the coverage matrix. + +Pure functions that tally AC coverage. An AC contributes to ``covered_acs`` only when +it is Active and covered, so deprecated ACs never inflate ``coverage_pct``. +""" + +from living_doc_service_coverage_matrix.model.coverage_item import ( + AcCoverage, + Summary, + TopSummary, + UserStoryCoverage, +) + +ACTIVE_STATE = "Active" +COVERED = "covered" + + +def _coverage_pct(covered: int, active: int) -> float | None: + """Return covered/active as a percentage rounded to 1 dp, or None when active is 0.""" + if active == 0: + return None + return round(covered / active * 100, 1) + + +def _is_active_covered(ac: AcCoverage) -> bool: + """True when an AC is Active and has at least one covering test.""" + return ac.state == ACTIVE_STATE and ac.coverage.status == COVERED + + +def compute_us_summary(acceptance_criteria: list[AcCoverage]) -> Summary: + """Compute the coverage summary for a single User Story's acceptance criteria.""" + total = len(acceptance_criteria) + active = sum(1 for ac in acceptance_criteria if ac.state == ACTIVE_STATE) + covered = sum(1 for ac in acceptance_criteria if _is_active_covered(ac)) + return Summary( + total_acs=total, + active_acs=active, + covered_acs=covered, + coverage_pct=_coverage_pct(covered, active), + ) + + +def compute_summary(user_stories: list[UserStoryCoverage]) -> TopSummary: + """Compute the top-level coverage summary across all User Stories.""" + all_acs = [ac for us in user_stories for ac in us.acceptance_criteria] + total = len(all_acs) + active = sum(1 for ac in all_acs if ac.state == ACTIVE_STATE) + covered = sum(1 for ac in all_acs if _is_active_covered(ac)) + return TopSummary( + total_user_stories=len(user_stories), + total_acs=total, + active_acs=active, + covered_acs=covered, + coverage_pct=_coverage_pct(covered, active), + ) diff --git a/packages/services/coverage_matrix/tests/__init__.py b/packages/services/coverage_matrix/tests/__init__.py new file mode 100644 index 0000000..f8d2e07 --- /dev/null +++ b/packages/services/coverage_matrix/tests/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Unit tests for the coverage-matrix service.""" diff --git a/packages/services/coverage_matrix/tests/fixtures/golden/doc_source.json b/packages/services/coverage_matrix/tests/fixtures/golden/doc_source.json new file mode 100644 index 0000000..348263f --- /dev/null +++ b/packages/services/coverage_matrix/tests/fixtures/golden/doc_source.json @@ -0,0 +1,90 @@ +{ + "items": [ + { + "id": "absa-group/aul-ui/US-1", + "title": "User Login", + "state": "active", + "tags": [], + "url": "https://github.com/absa-group/aul-ui/issues/2", + "timestamps": null, + "description": "As a user, I want to log in so that I can access the application.", + "business_value": [], + "preconditions": [], + "acceptance_criteria": [ + { + "id": "US-1-01", + "state": "Active", + "version": "v1.0.0", + "description": "User can log in with valid credentials." + }, + { + "id": "US-1-02", + "state": "Active", + "version": "v1.0.0", + "description": "Login button is disabled until both fields are filled." + }, + { + "id": "US-1-03", + "state": "Deprecated", + "version": "v0.9.0", + "description": "Legacy basic-auth login (removed)." + } + ] + }, + { + "id": "absa-group/aul-ui/US-2", + "title": "View Dashboard", + "state": "active", + "tags": [], + "url": null, + "timestamps": null, + "description": "As a user, I want a dashboard overview.", + "business_value": [], + "preconditions": [], + "acceptance_criteria": [ + { + "id": "US-2-01", + "state": "Active", + "version": "v1.0.0", + "description": "User can see accessible domains on the dashboard." + }, + { + "id": "US-2-02", + "state": "Active", + "version": "v1.0.0", + "description": "Dashboard shows an empty state when no domains are accessible." + } + ] + }, + { + "id": "absa-group/aul-ui/US-7", + "title": "Delete Domain", + "state": "active", + "tags": [], + "url": null, + "timestamps": null, + "description": "As a user, I want to delete a domain.", + "business_value": [], + "preconditions": [], + "acceptance_criteria": [] + } + ], + "metadata": { + "producer": { + "name": "AbsaOSS/living-doc-collector-gh", + "version": "0.1.1", + "build": null + }, + "source": { + "systems": [ + "GitHub" + ], + "repositories": [ + "absa-group/aul-ui" + ], + "organization": "absa-group", + "enterprise": null + } + }, + "warnings": [] +} diff --git a/packages/services/coverage_matrix/tests/fixtures/golden/expected_coverage_matrix.json b/packages/services/coverage_matrix/tests/fixtures/golden/expected_coverage_matrix.json new file mode 100644 index 0000000..321c8e0 --- /dev/null +++ b/packages/services/coverage_matrix/tests/fixtures/golden/expected_coverage_matrix.json @@ -0,0 +1,213 @@ +{ + "generated_at": "PLACEHOLDER", + "schema_version": "coverage-matrix-v1.0.0", + "stale_ac_refs": [ + { + "scenario_id": "absa-group/aul-ui/user_login.feature/references-unknown-ac", + "scenario_name": "Scenario referencing an AC that no longer exists", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "stale_ac_id": "US-1-99", + "us_id": "US-1" + } + ], + "summary": { + "active_acs": 4, + "coverage_pct": 75.0, + "covered_acs": 3, + "total_acs": 5, + "total_user_stories": 3 + }, + "unlinked_tests": [ + { + "ac_ids": [], + "id": "absa-group/aul-ui/orphan.feature/scenario-without-us", + "scenario_name": "Scenario not linked to any user story", + "source": { + "file": "orphan.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "us_id": null + }, + { + "ac_ids": [ + "US-99-01" + ], + "id": "absa-group/aul-ui/unknown.feature/scenario-for-unknown-us", + "scenario_name": "Scenario for a user story not in the doc input", + "source": { + "file": "unknown.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "us_id": "US-99" + } + ], + "user_stories": [ + { + "acceptance_criteria": [ + { + "coverage": { + "status": "covered", + "test_count": 2, + "tests": [ + { + "id": "absa-group/aul-ui/user_login.feature/user-logs-in-with-valid-credentials", + "scenario_name": "User logs in with valid credentials", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "tags": [ + "Regression" + ] + }, + { + "id": "absa-group/aul-ui/user_login.feature/login-button-disabled", + "scenario_name": "Login button stays disabled until fields are filled", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "tags": [ + "Smoke" + ] + } + ] + }, + "description": "User can log in with valid credentials.", + "id": "US-1-01", + "state": "Active", + "version": "v1.0.0" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/aul-ui/user_login.feature/user-logs-in-with-valid-credentials", + "scenario_name": "User logs in with valid credentials", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "Login button is disabled until both fields are filled.", + "id": "US-1-02", + "state": "Active", + "version": "v1.0.0" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/aul-ui/user_login.feature/legacy-basic-auth-login", + "scenario_name": "Legacy basic-auth login", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "tags": [ + "Regression", + "skip" + ] + } + ] + }, + "description": "Legacy basic-auth login (removed).", + "id": "US-1-03", + "state": "Deprecated", + "version": "v0.9.0" + } + ], + "full_id": "absa-group/aul-ui/US-1", + "id": "US-1", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 100.0, + "covered_acs": 2, + "total_acs": 3 + }, + "title": "User Login" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/aul-ui/dashboard.feature/user-sees-accessible-domains", + "scenario_name": "User sees accessible domains on the dashboard", + "source": { + "file": "dashboard.feature", + "org": "absa-group", + "repo": "aul-ui" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "User can see accessible domains on the dashboard.", + "id": "US-2-01", + "state": "Active", + "version": "v1.0.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Dashboard shows an empty state when no domains are accessible.", + "id": "US-2-02", + "state": "Active", + "version": "v1.0.0" + } + ], + "full_id": "absa-group/aul-ui/US-2", + "id": "US-2", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 50.0, + "covered_acs": 1, + "total_acs": 2 + }, + "title": "View Dashboard" + }, + { + "acceptance_criteria": [], + "full_id": "absa-group/aul-ui/US-7", + "id": "US-7", + "state": "active", + "summary": { + "active_acs": 0, + "coverage_pct": null, + "covered_acs": 0, + "total_acs": 0 + }, + "title": "Delete Domain" + } + ] +} diff --git a/packages/services/coverage_matrix/tests/fixtures/golden/ui_tests.json b/packages/services/coverage_matrix/tests/fixtures/golden/ui_tests.json new file mode 100644 index 0000000..0f60d79 --- /dev/null +++ b/packages/services/coverage_matrix/tests/fixtures/golden/ui_tests.json @@ -0,0 +1,146 @@ +{ + "items": [ + { + "id": "absa-group/aul-ui/user_login.feature/user-logs-in-with-valid-credentials", + "us_id": "US-1", + "ac_ids": [ + "US-1-01", + "US-1-02" + ], + "scenario_name": "User logs in with valid credentials", + "scenario_type": "Scenario", + "tags": [ + "Regression" + ], + "steps": [], + "source": { + "org": "absa-group", + "repo": "aul-ui", + "file": "user_login.feature" + } + }, + { + "id": "absa-group/aul-ui/user_login.feature/legacy-basic-auth-login", + "us_id": "US-1", + "ac_ids": [ + "US-1-03" + ], + "scenario_name": "Legacy basic-auth login", + "scenario_type": "Scenario", + "tags": [ + "Regression", + "skip" + ], + "steps": [], + "source": { + "org": "absa-group", + "repo": "aul-ui", + "file": "user_login.feature" + } + }, + { + "id": "absa-group/aul-ui/user_login.feature/login-button-disabled", + "us_id": "US-1", + "ac_ids": [ + "US-1-01" + ], + "scenario_name": "Login button stays disabled until fields are filled", + "scenario_type": "Scenario", + "tags": [ + "Smoke" + ], + "steps": [], + "source": { + "org": "absa-group", + "repo": "aul-ui", + "file": "user_login.feature" + } + }, + { + "id": "absa-group/aul-ui/user_login.feature/references-unknown-ac", + "us_id": "US-1", + "ac_ids": [ + "US-1-99" + ], + "scenario_name": "Scenario referencing an AC that no longer exists", + "scenario_type": "Scenario", + "tags": [ + "Regression" + ], + "steps": [], + "source": { + "org": "absa-group", + "repo": "aul-ui", + "file": "user_login.feature" + } + }, + { + "id": "absa-group/aul-ui/dashboard.feature/user-sees-accessible-domains", + "us_id": "US-2", + "ac_ids": [ + "US-2-01" + ], + "scenario_name": "User sees accessible domains on the dashboard", + "scenario_type": "Scenario", + "tags": [ + "Regression" + ], + "steps": [], + "source": { + "org": "absa-group", + "repo": "aul-ui", + "file": "dashboard.feature" + } + }, + { + "id": "absa-group/aul-ui/orphan.feature/scenario-without-us", + "us_id": null, + "ac_ids": [], + "scenario_name": "Scenario not linked to any user story", + "scenario_type": "Scenario", + "tags": [], + "steps": [], + "source": { + "org": "absa-group", + "repo": "aul-ui", + "file": "orphan.feature" + } + }, + { + "id": "absa-group/aul-ui/unknown.feature/scenario-for-unknown-us", + "us_id": "US-99", + "ac_ids": [ + "US-99-01" + ], + "scenario_name": "Scenario for a user story not in the doc input", + "scenario_type": "Scenario", + "tags": [ + "Regression" + ], + "steps": [], + "source": { + "org": "absa-group", + "repo": "aul-ui", + "file": "unknown.feature" + } + } + ], + "metadata": { + "producer": { + "name": "AbsaOSS/living-doc-collector-gh", + "version": "0.1.1", + "build": null + }, + "source": { + "systems": [ + "GitHub" + ], + "repositories": [ + "absa-group/aul-ui" + ], + "organization": "absa-group", + "enterprise": null + } + }, + "warnings": [] +} diff --git a/packages/services/coverage_matrix/tests/integration/__init__.py b/packages/services/coverage_matrix/tests/integration/__init__.py new file mode 100644 index 0000000..939961c --- /dev/null +++ b/packages/services/coverage_matrix/tests/integration/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Integration tests for the coverage-matrix service.""" diff --git a/packages/services/coverage_matrix/tests/integration/test_golden_files.py b/packages/services/coverage_matrix/tests/integration/test_golden_files.py new file mode 100644 index 0000000..be91a6e --- /dev/null +++ b/packages/services/coverage_matrix/tests/integration/test_golden_files.py @@ -0,0 +1,76 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Integration test for the coverage-matrix service using the golden fixtures.""" + +import json +from pathlib import Path + +from living_doc_service_coverage_matrix.service import run_service + +FIXTURES = Path(__file__).parent.parent / "fixtures" / "golden" + + +def test_golden_coverage_matrix(tmp_path): + """Run the full pipeline and compare against the golden expected output.""" + doc_file = FIXTURES / "doc_source.json" + tests_file = FIXTURES / "ui_tests.json" + expected_file = FIXTURES / "expected_coverage_matrix.json" + out_file = tmp_path / "coverage-matrix.json" + + run_service(str(doc_file), str(tests_file), str(out_file), {}) + + with open(expected_file, "r", encoding="utf-8") as f: + expected = json.load(f) + with open(out_file, "r", encoding="utf-8") as f: + actual = json.load(f) + + # generated_at is dynamic; normalise before comparing. + assert actual["generated_at"] + actual["generated_at"] = "PLACEHOLDER" + + assert actual == expected, "Output does not match the golden expected_coverage_matrix.json" + + +def test_golden_summary_and_buckets(tmp_path): + """Assert the key coverage facts produced from the golden fixtures.""" + doc_file = FIXTURES / "doc_source.json" + tests_file = FIXTURES / "ui_tests.json" + out_file = tmp_path / "coverage-matrix.json" + + matrix = run_service(str(doc_file), str(tests_file), str(out_file), {}) + + summary = matrix.summary + assert summary.total_user_stories == 3 + assert summary.total_acs == 5 + assert summary.active_acs == 4 + assert summary.covered_acs == 3 + assert summary.coverage_pct == 75.0 + + us_by_id = {us.id: us for us in matrix.user_stories} + + # US-1: the deprecated AC is covered but excluded from coverage_pct. + us1 = us_by_id["US-1"] + us1_cov = {ac.id: ac.coverage for ac in us1.acceptance_criteria} + assert us1_cov["US-1-01"].status == "covered" + assert us1_cov["US-1-01"].test_count == 2 + assert us1_cov["US-1-02"].status == "covered" + assert us1_cov["US-1-03"].status == "covered" + assert us1.summary.coverage_pct == 100.0 + + # US-2: one covered, one not covered. + us2 = us_by_id["US-2"] + us2_cov = {ac.id: ac.coverage.status for ac in us2.acceptance_criteria} + assert us2_cov == {"US-2-01": "covered", "US-2-02": "not_covered"} + + # US-7: no ACs -> null coverage_pct. + us7 = us_by_id["US-7"] + assert us7.acceptance_criteria == [] + assert us7.summary.coverage_pct is None + + # Two unlinked scenarios (null us_id and unresolved US-99) and one stale AC ref. + unlinked_us_ids = sorted(str(t.us_id) for t in matrix.unlinked_tests) + assert unlinked_us_ids == ["None", "US-99"] + assert len(matrix.stale_ac_refs) == 1 + assert matrix.stale_ac_refs[0].stale_ac_id == "US-1-99" + assert matrix.stale_ac_refs[0].us_id == "US-1" + diff --git a/packages/services/coverage_matrix/tests/test_loader.py b/packages/services/coverage_matrix/tests/test_loader.py new file mode 100644 index 0000000..1d0a959 --- /dev/null +++ b/packages/services/coverage_matrix/tests/test_loader.py @@ -0,0 +1,76 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Unit tests for the loader module.""" + +import json + +import pytest + +from living_doc_core.errors import FileIOError, InvalidInputError + +from living_doc_service_coverage_matrix.loader import load_doc_input, load_tests_input + + +def _write(path, payload): + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f) + + +def test_load_doc_input_bare_array(tmp_path): + doc_file = tmp_path / "doc.json" + _write(doc_file, [{"id": "org/repo/US-1", "acceptance_criteria": []}]) + + items = load_doc_input(str(doc_file)) + + assert len(items) == 1 + assert items[0]["id"] == "org/repo/US-1" + + +def test_load_doc_input_envelope(tmp_path): + doc_file = tmp_path / "doc.json" + _write(doc_file, {"items": [{"id": "org/repo/US-1", "acceptance_criteria": []}], "metadata": {}}) + + items = load_doc_input(str(doc_file)) + + assert len(items) == 1 + assert items[0]["id"] == "org/repo/US-1" + + +def test_load_doc_input_invalid_shape(tmp_path): + doc_file = tmp_path / "doc.json" + _write(doc_file, {"no_items": True}) + + with pytest.raises(InvalidInputError): + load_doc_input(str(doc_file)) + + +def test_load_doc_input_missing_file(tmp_path): + with pytest.raises(FileIOError): + load_doc_input(str(tmp_path / "missing.json")) + + +def test_load_tests_input_envelope(tmp_path): + tests_file = tmp_path / "tests.json" + _write(tests_file, {"items": [{"id": "s1", "us_id": "US-1"}]}) + + items = load_tests_input(str(tests_file)) + + assert len(items) == 1 + assert items[0]["us_id"] == "US-1" + + +def test_load_tests_input_missing_items(tmp_path): + tests_file = tmp_path / "tests.json" + _write(tests_file, [{"id": "s1"}]) + + with pytest.raises(InvalidInputError): + load_tests_input(str(tests_file)) + + +def test_load_tests_input_malformed_json(tmp_path): + tests_file = tmp_path / "tests.json" + with open(tests_file, "w", encoding="utf-8") as f: + f.write("{ not valid json ") + + with pytest.raises(InvalidInputError): + load_tests_input(str(tests_file)) diff --git a/packages/services/coverage_matrix/tests/test_matcher.py b/packages/services/coverage_matrix/tests/test_matcher.py new file mode 100644 index 0000000..2045932 --- /dev/null +++ b/packages/services/coverage_matrix/tests/test_matcher.py @@ -0,0 +1,160 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Unit tests for the matcher module.""" + +from living_doc_service_coverage_matrix.matcher import build_coverage_matrix + +GENERATED_AT = "2026-06-30T00:00:00+00:00" + + +def _us(num, acs, state="active", title="Title"): + return { + "id": f"org/repo/{num}", + "title": title, + "state": state, + "acceptance_criteria": acs, + } + + +def _ac(ac_id, state="Active", version="v1.0.0", description="desc"): + return {"id": ac_id, "state": state, "version": version, "description": description} + + +def _scenario(scenario_id, us_id, ac_ids, name="scenario", tags=None, source=None): + return { + "id": scenario_id, + "us_id": us_id, + "ac_ids": ac_ids, + "scenario_name": name, + "tags": tags if tags is not None else ["Regression"], + "source": source if source is not None else {"org": "org", "repo": "repo", "file": "f.feature"}, + } + + +def test_covered_and_not_covered(): + doc = [_us("US-1", [_ac("US-1-01"), _ac("US-1-02")])] + tests = [_scenario("s1", "US-1", ["US-1-01"])] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + us = matrix.user_stories[0] + assert us.id == "US-1" + assert us.full_id == "org/repo/US-1" + ac1, ac2 = us.acceptance_criteria + assert ac1.coverage.status == "covered" + assert ac1.coverage.test_count == 1 + assert ac1.coverage.tests[0].id == "s1" + assert ac2.coverage.status == "not_covered" + assert ac2.coverage.test_count == 0 + assert ac2.coverage.tests == [] + + +def test_us_summary_counts(): + doc = [_us("US-1", [_ac("US-1-01"), _ac("US-1-02")])] + tests = [_scenario("s1", "US-1", ["US-1-01"])] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + summary = matrix.user_stories[0].summary + assert summary.total_acs == 2 + assert summary.active_acs == 2 + assert summary.covered_acs == 1 + assert summary.coverage_pct == 50.0 + + +def test_deprecated_ac_excluded_from_pct(): + doc = [_us("US-1", [_ac("US-1-01"), _ac("US-1-02", state="Deprecated")])] + tests = [_scenario("s1", "US-1", ["US-1-01", "US-1-02"])] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + summary = matrix.user_stories[0].summary + assert summary.total_acs == 2 + assert summary.active_acs == 1 + assert summary.covered_acs == 1 + assert summary.coverage_pct == 100.0 + + +def test_unlinked_when_us_id_null(): + doc = [_us("US-1", [_ac("US-1-01")])] + tests = [_scenario("s1", None, ["US-1-01"])] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + assert matrix.user_stories[0].acceptance_criteria[0].coverage.status == "not_covered" + assert len(matrix.unlinked_tests) == 1 + assert matrix.unlinked_tests[0].id == "s1" + assert matrix.unlinked_tests[0].us_id is None + + +def test_unlinked_when_us_id_unresolved(): + doc = [_us("US-1", [_ac("US-1-01")])] + tests = [_scenario("s1", "US-99", ["US-99-01"])] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + assert len(matrix.unlinked_tests) == 1 + assert matrix.unlinked_tests[0].us_id == "US-99" + assert matrix.stale_ac_refs == [] + + +def test_stale_ac_ref_recorded(): + doc = [_us("US-1", [_ac("US-1-01")])] + tests = [_scenario("s1", "US-1", ["US-1-01", "US-1-99"])] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + assert matrix.user_stories[0].acceptance_criteria[0].coverage.status == "covered" + assert len(matrix.stale_ac_refs) == 1 + stale = matrix.stale_ac_refs[0] + assert stale.scenario_id == "s1" + assert stale.us_id == "US-1" + assert stale.stale_ac_id == "US-1-99" + + +def test_multiple_tests_for_one_ac(): + doc = [_us("US-1", [_ac("US-1-01")])] + tests = [ + _scenario("s1", "US-1", ["US-1-01"]), + _scenario("s2", "US-1", ["US-1-01"]), + ] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + coverage = matrix.user_stories[0].acceptance_criteria[0].coverage + assert coverage.test_count == 2 + assert {t.id for t in coverage.tests} == {"s1", "s2"} + + +def test_top_summary_aggregation(): + doc = [ + _us("US-1", [_ac("US-1-01"), _ac("US-1-02")]), + _us("US-2", [_ac("US-2-01")]), + ] + tests = [_scenario("s1", "US-1", ["US-1-01"])] + + matrix = build_coverage_matrix(doc, tests, GENERATED_AT) + + summary = matrix.summary + assert summary.total_user_stories == 2 + assert summary.total_acs == 3 + assert summary.active_acs == 3 + assert summary.covered_acs == 1 + assert summary.coverage_pct == 33.3 + + +def test_coverage_pct_null_when_no_active_acs(): + doc = [_us("US-1", [_ac("US-1-01", state="Deprecated")])] + + matrix = build_coverage_matrix(doc, [], GENERATED_AT) + + assert matrix.summary.coverage_pct is None + assert matrix.user_stories[0].summary.coverage_pct is None + + +def test_schema_version_and_timestamp(): + matrix = build_coverage_matrix([], [], GENERATED_AT) + + assert matrix.schema_version == "coverage-matrix-v1.0.0" + assert matrix.generated_at == GENERATED_AT + assert matrix.summary.total_user_stories == 0 diff --git a/packages/services/coverage_matrix/tests/test_service.py b/packages/services/coverage_matrix/tests/test_service.py new file mode 100644 index 0000000..39cc862 --- /dev/null +++ b/packages/services/coverage_matrix/tests/test_service.py @@ -0,0 +1,106 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Unit tests for the service orchestration module.""" + +import json + +import pytest + +from living_doc_service_coverage_matrix.service import CoverageThresholdError, run_service + + +def _write(path, payload): + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f) + + +def _doc_payload(): + return { + "items": [ + { + "id": "org/repo/US-1", + "title": "User Login", + "state": "active", + "acceptance_criteria": [ + {"id": "US-1-01", "state": "Active", "version": "v1.0.0", "description": "a"}, + {"id": "US-1-02", "state": "Active", "version": "v1.0.0", "description": "b"}, + ], + } + ] + } + + +def _tests_payload(): + return { + "items": [ + { + "id": "s1", + "us_id": "US-1", + "ac_ids": ["US-1-01"], + "scenario_name": "scenario one", + "tags": ["Regression"], + "source": {"org": "org", "repo": "repo", "file": "f.feature"}, + } + ] + } + + +def test_run_service_writes_output(tmp_path): + doc_file = tmp_path / "doc.json" + tests_file = tmp_path / "tests.json" + out_file = tmp_path / "coverage-matrix.json" + _write(doc_file, _doc_payload()) + _write(tests_file, _tests_payload()) + + run_service(str(doc_file), str(tests_file), str(out_file), {}) + + assert out_file.exists() + with open(out_file, "r", encoding="utf-8") as f: + data = json.load(f) + + assert data["schema_version"] == "coverage-matrix-v1.0.0" + assert data["summary"]["total_user_stories"] == 1 + assert data["summary"]["covered_acs"] == 1 + assert data["summary"]["coverage_pct"] == 50.0 + assert data["user_stories"][0]["id"] == "US-1" + + +def test_run_service_skips_invalid_user_story(tmp_path): + doc_file = tmp_path / "doc.json" + tests_file = tmp_path / "tests.json" + out_file = tmp_path / "out.json" + payload = _doc_payload() + payload["items"].append({"title": "no id"}) + payload["items"].append({"id": "org/repo/US-2"}) # missing acceptance_criteria + _write(doc_file, payload) + _write(tests_file, _tests_payload()) + + matrix = run_service(str(doc_file), str(tests_file), str(out_file), {}) + + assert matrix.summary.total_user_stories == 1 + + +def test_run_service_fail_under_raises(tmp_path): + doc_file = tmp_path / "doc.json" + tests_file = tmp_path / "tests.json" + out_file = tmp_path / "out.json" + _write(doc_file, _doc_payload()) + _write(tests_file, _tests_payload()) + + with pytest.raises(CoverageThresholdError): + run_service(str(doc_file), str(tests_file), str(out_file), {"fail_under": 80.0}) + + # Output is still written before the threshold check. + assert out_file.exists() + + +def test_run_service_fail_under_passes(tmp_path): + doc_file = tmp_path / "doc.json" + tests_file = tmp_path / "tests.json" + out_file = tmp_path / "out.json" + _write(doc_file, _doc_payload()) + _write(tests_file, _tests_payload()) + + matrix = run_service(str(doc_file), str(tests_file), str(out_file), {"fail_under": 50.0}) + + assert matrix.summary.coverage_pct == 50.0 diff --git a/packages/services/coverage_matrix/tests/test_summary.py b/packages/services/coverage_matrix/tests/test_summary.py new file mode 100644 index 0000000..792f544 --- /dev/null +++ b/packages/services/coverage_matrix/tests/test_summary.py @@ -0,0 +1,71 @@ +# Copyright 2026 ABSA Group Limited. Apache License, Version 2.0. + +"""Unit tests for the summary module.""" + +from living_doc_service_coverage_matrix.model.coverage_item import ( + AcCoverage, + Coverage, + UserStoryCoverage, +) +from living_doc_service_coverage_matrix.summary import compute_summary, compute_us_summary + + +def _ac(state, status): + return AcCoverage( + id="US-1-01", + state=state, + version="v1.0.0", + description="desc", + coverage=Coverage(status=status, test_count=1 if status == "covered" else 0, tests=[]), + ) + + +def test_compute_us_summary_mixed(): + acs = [ + _ac("Active", "covered"), + _ac("Active", "not_covered"), + _ac("Deprecated", "covered"), + ] + + summary = compute_us_summary(acs) + + assert summary.total_acs == 3 + assert summary.active_acs == 2 + assert summary.covered_acs == 1 + assert summary.coverage_pct == 50.0 + + +def test_compute_us_summary_empty(): + summary = compute_us_summary([]) + + assert summary.total_acs == 0 + assert summary.active_acs == 0 + assert summary.covered_acs == 0 + assert summary.coverage_pct is None + + +def test_compute_summary_aggregates_user_stories(): + us1 = UserStoryCoverage( + id="US-1", + full_id="org/repo/US-1", + title="t", + state="active", + summary=compute_us_summary([_ac("Active", "covered")]), + acceptance_criteria=[_ac("Active", "covered")], + ) + us2 = UserStoryCoverage( + id="US-2", + full_id="org/repo/US-2", + title="t", + state="active", + summary=compute_us_summary([_ac("Active", "not_covered")]), + acceptance_criteria=[_ac("Active", "not_covered")], + ) + + summary = compute_summary([us1, us2]) + + assert summary.total_user_stories == 2 + assert summary.total_acs == 2 + assert summary.active_acs == 2 + assert summary.covered_acs == 1 + assert summary.coverage_pct == 50.0 From ffda4787853ca1a46924b7c77279e2d90809ea8c Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 30 Jun 2026 15:59:07 +0200 Subject: [PATCH 2/2] Refactor code structure for improved readability and maintainability --- .gitignore | 3 + CHANGELOG.md | 9 + DEVELOPER.md | 15 +- README.md | 12 + coverage-matrix.json | 1910 +++++++++++++++++ docs/architecture.md | 79 +- docs/contracts.md | 70 +- .../tests/integration/test_golden_files.py | 1 - 8 files changed, 2081 insertions(+), 18 deletions(-) create mode 100644 coverage-matrix.json diff --git a/.gitignore b/.gitignore index 267521d..c400bb4 100644 --- a/.gitignore +++ b/.gitignore @@ -211,3 +211,6 @@ outputs/ .DS_Store doc-issues.json pdf_ready.json +ui-tests.json +doc-source.json +spec.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6028021..5e13950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CHANGELOG.md following Keep a Changelog format - Copyright header template for Python files - Updated README.md with project overview and quickstart +- `packages/services/coverage_matrix` — new `living-doc-service-coverage-matrix` package + - `loader.py` — I/O boundary: loads `doc-source.json` (bare array or envelope) and `ui-tests.json` + - `matcher.py` — pure function: builds `CoverageMatrix` from parsed doc and test lists + - `summary.py` — pure tallying: `compute_us_summary()` / `compute_summary()` (deprecated ACs excluded from `coverage_pct`) + - `model/coverage_item.py` — output dataclasses serialising to `coverage-matrix.json` + - `service.py` — orchestration with optional `--fail-under` threshold enforcement + - `schema/coverage-matrix-v1.0.0-schema.json` — JSON Schema draft-07 for the output contract +- `living-doc coverage-matrix` CLI command (`apps/cli/src/living_doc_cli/commands/coverage_matrix.py`) +- `make py-qa-coverage` target for the new service package ### Changed diff --git a/DEVELOPER.md b/DEVELOPER.md index 9d303b7..c0a84e8 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -39,6 +39,7 @@ pip install -e packages/core[dev] pip install -e packages/datasets_pdf[dev] pip install -e packages/adapters/collector_gh[dev] pip install -e packages/services/normalize_issues[dev] +pip install -e packages/services/coverage_matrix[dev] pip install -e "apps/cli[dev]" ``` @@ -53,6 +54,7 @@ pip install -e "apps/cli[dev]" | `packages/datasets_pdf` | `living-doc-datasets-pdf` | Pydantic models and JSON schemas for PDF contracts | | `packages/adapters/collector_gh` | `living-doc-adapter-collector-gh` | Detector and parser for collector-gh output | | `packages/services/normalize_issues` | `living-doc-service-normalize-issues` | Issue normalization service | +| `packages/services/coverage_matrix` | `living-doc-service-coverage-matrix` | AC-level test coverage matrix generator | | `apps/cli` | `living-doc-cli` | CLI entry point (`living-doc` command) | Each package has its own `pyproject.toml`, `src/` layout, and `tests/` directory. @@ -103,6 +105,7 @@ make py-qa-core # packages/core make py-qa-datasets-pdf # packages/datasets_pdf make py-qa-collector-gh # packages/adapters/collector_gh make py-qa-normalize # packages/services/normalize_issues +make py-qa-coverage # packages/services/coverage_matrix make py-qa-cli # apps/cli ``` @@ -151,7 +154,7 @@ pylint src/living_doc_core/json_utils.py From the **repository root**: ```shell -for pkg in packages/core packages/datasets_pdf packages/adapters/collector_gh packages/services/normalize_issues apps/cli; do +for pkg in packages/core packages/datasets_pdf packages/adapters/collector_gh packages/services/normalize_issues packages/services/coverage_matrix apps/cli; do echo "=== Pylint: $pkg ===" (cd "$pkg" && pylint $(git ls-files '*.py' | grep -v '^tests/')) done @@ -211,7 +214,7 @@ mypy . From the **repository root**: ```shell -for pkg in packages/core packages/datasets_pdf packages/adapters/collector_gh packages/services/normalize_issues apps/cli; do +for pkg in packages/core packages/datasets_pdf packages/adapters/collector_gh packages/services/normalize_issues packages/services/coverage_matrix apps/cli; do echo "=== mypy: $pkg ===" (cd "$pkg" && mypy .) done @@ -242,7 +245,7 @@ pytest --cov=src -v tests/ --cov-fail-under=80 From the **repository root**: ```shell -for pkg in packages/core packages/datasets_pdf packages/adapters/collector_gh packages/services/normalize_issues apps/cli; do +for pkg in packages/core packages/datasets_pdf packages/adapters/collector_gh packages/services/normalize_issues packages/services/coverage_matrix apps/cli; do echo "=== Tests: $pkg ===" (cd "$pkg" && pytest --cov=src -v tests/ --cov-fail-under=80) done @@ -304,6 +307,12 @@ living-doc normalize-issues \ --source auto \ --document-title "Sprint 42 Report" \ --document-version "1.0.0" + +living-doc coverage-matrix \ + --doc-input doc-source.json \ + --tests-input ui-tests.json \ + --output coverage-matrix.json \ + --fail-under 80 ``` ## Branch Naming Convention (PID:H-1) diff --git a/README.md b/README.md index 9c68b8a..7264f60 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,13 @@ living-doc normalize-issues \ --source auto \ --document-title "Sprint 42 Report" \ --document-version "1.0.0" + +# Generate an AC-level test coverage matrix +living-doc coverage-matrix \ + --doc-input doc-source.json \ + --tests-input ui-tests.json \ + --output coverage-matrix.json \ + --fail-under 80 ``` ## Documentation @@ -78,6 +85,11 @@ Converts collector output (`doc-issues.json`) into PDF-ready canonical JSON (`pd - [Recipe: Local usage](docs/recipes/local-normalize-issues.md) — Run the CLI on your machine - [Recipe: GitHub Actions](docs/recipes/github-actions-normalize-issues.md) — CI/CD workflow integration +### `coverage-matrix` +Cross-references a `doc-source.json` (User Stories + acceptance criteria) with a `ui-tests.json` (E2E test scenarios) and produces `coverage-matrix.json`: an AC-level test coverage matrix per User Story. + +- [Package README](packages/services/coverage_matrix/README.md) — Matching logic, CLI reference, module layout + ## License Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for full details. diff --git a/coverage-matrix.json b/coverage-matrix.json new file mode 100644 index 0000000..0acf2fa --- /dev/null +++ b/coverage-matrix.json @@ -0,0 +1,1910 @@ +{ + "generated_at": "2026-06-30T13:57:10.613330+00:00", + "schema_version": "coverage-matrix-v1.0.0", + "stale_ac_refs": [], + "summary": { + "active_acs": 114, + "coverage_pct": 8.8, + "covered_acs": 10, + "total_acs": 121, + "total_user_stories": 31 + }, + "unlinked_tests": [], + "user_stories": [ + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can see the domains they have access to on the dashboard.", + "id": "US-2-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Dashboard displays an empty state message when no domains are accessible to the user.", + "id": "US-2-02", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each domain card displays: name, creation date, primary owner, and classification.", + "id": "US-2-03", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The dashboard shows three separate domain carousels: recently visited domains, primary owned domains, and draft domains.", + "id": "US-2-04", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each domain card on the dashboard exposes a More Actions menu with context-sensitive options: owned domains additionally show \"Grant access\" and \"Edit domain\"; all cards show \"View data feeds\", \"View statistics\", \"View questions\", and \"Survivorship\".", + "id": "US-2-05", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each domain card on the dashboard shows the count of pending access requests and open questions for that domain.", + "id": "US-2-06", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A \"View all domains\" button on the dashboard navigates the user to the full Domain Catalogue page.", + "id": "US-2-07", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The dashboard toolbar provides \"Create Domain\" and \"Import domain\" buttons that launch the respective flows without leaving the dashboard.", + "id": "US-2-08", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-2", + "id": "US-2", + "state": "active", + "summary": { + "active_acs": 8, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 8 + }, + "title": "View Dashboard" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can edit fields such as owner tab, type of domain, target dataset, classification, Availability and schedule.", + "id": "US-9-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User cannot edit fields such domain name, type of domain and country code after creation of data feed.", + "id": "US-9-02", + "state": "Active", + "version": "v1.5.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-9", + "id": "US-9", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 2 + }, + "title": "Edit Data Feeds" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can access the data feeds details.", + "id": "US-18-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The user can see all data feeds across all domains, regardless of ownership.", + "id": "US-18-02", + "state": "Active", + "version": "v1.4.3" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Data feed detail view displays: name, description, availability status, and tabs for About, Data sources, Mapping, and Version management.", + "id": "US-18-03", + "state": "Active", + "version": "v1.8.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-18", + "id": "US-18", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "View Data Feed" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The user must capture all mandatory fields.", + "id": "US-3-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The user can import the data source from a file or from a Glue table when.", + "id": "US-3-02", + "state": "Active", + "version": "v1.5.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-3", + "id": "US-3", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 2 + }, + "title": "Add Data Feeds" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can delete data feeds.", + "id": "US-6-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User is prompted with a confirmation dialog before data feed deletion.", + "id": "US-6-02", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user who does not own the data feed cannot delete it.", + "id": "US-6-03", + "state": "Active", + "version": "v1.8.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-6", + "id": "US-6", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "Delete Data Feeds" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can search data feed by name or partial name.", + "id": "US-12-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each data feed from search results displays its details such as data feed name, description, and the domain it belongs to.", + "id": "US-12-02", + "state": "Active", + "version": "v1.5.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-12", + "id": "US-12", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 2 + }, + "title": "Search Data Feeds" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The user can directly approve or reject the domain access request.", + "id": "US-24-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Only the domain owner can directly grant or reject access.", + "id": "US-24-02", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "After approving or rejecting, the requester receives a status update notification.", + "id": "US-24-03", + "state": "Active", + "version": "v1.8.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-24", + "id": "US-24", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "Domain Access - Direct Granting Access" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user who is not the domain owner can open the Access tab on the Domain Detail page and see a \"Request access\" button.", + "id": "US-27-01", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "After submitting an access request, the user receives an on-screen confirmation and the request appears in their Domain Access Rights page with status \"Pending\".", + "id": "US-27-02", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user who has already submitted a pending request for a domain cannot submit a duplicate request for the same domain.", + "id": "US-27-03", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A domain owner cannot request access to their own domain; the \"Request access\" button is not shown for owned domains.", + "id": "US-27-04", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-27", + "id": "US-27", + "state": "active", + "summary": { + "active_acs": 4, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 4 + }, + "title": "Request Access to Domain" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can view domain access request in states: Pending, Approved, and Rejected.", + "id": "US-23-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can manage pending requests by approving or rejecting them.", + "id": "US-23-02", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each item in the domain access request overview shows this information: domain name, domain version, requesting user, requested access level, and request timestamp.", + "id": "US-23-03", + "state": "Active", + "version": "v1.5.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-23", + "id": "US-23", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "Domain Access - Requests Approval" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user can open the \"My access rights to domains\" page from the Settings overlay and see a table listing all their access requests.", + "id": "US-28-01", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each row in the table displays: domain name, domain version, requested access level, request status, and time of request.", + "id": "US-28-02", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user can request access to a new domain directly from this page using the \"Request access\" button.", + "id": "US-28-03", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user can view the detail of any access request in the list using the \"View\" action.", + "id": "US-28-04", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "When the user has no access requests, the page displays an empty state.", + "id": "US-28-05", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-28", + "id": "US-28", + "state": "active", + "summary": { + "active_acs": 5, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 5 + }, + "title": "View My Domain Access Rights" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can search users with access by name or partial name.", + "id": "US-25-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each users from search results shows this information: user name and Ab number.", + "id": "US-25-02", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can search users among domains in primary of secondary ownership.", + "id": "US-25-03", + "state": "Active", + "version": "v1.5.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-25", + "id": "US-25", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "Domain Access - Overview of Users with Access" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/domain_create.feature/user-can-complete-the-create-domain-wizard-and-create-a-new-domain", + "scenario_name": "User can complete the Create Domain wizard and create a new domain", + "source": { + "file": "domain_create.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "User can create a new domain.", + "id": "US-26-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/domain_create.feature/invalid-cost-center-format-shows-an-error-message-in-the-create-domain-wizard", + "scenario_name": "Invalid cost center format shows an error message in the Create Domain wizard", + "source": { + "file": "domain_create.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "User can see an error message when entering information in the wrong format on cost center field.", + "id": "US-26-02", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/domain_create.feature/save-button-is-disabled-when-mandatory-fields-are-incomplete-on-the-about-step-o", + "scenario_name": "Save button is disabled when mandatory fields are incomplete on the About step of the Create Domain wizard", + "source": { + "file": "domain_create.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "Save button remains disabled until all mandatory fields are filled on create domain wizard {aspect} step. Aspect: About", + "id": "US-26-03", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The Create Domain wizard autosaves the user's progress; an autosave status indicator is visible and can be toggled on or off by the user.", + "id": "US-26-04", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The user can undo and redo changes made within the wizard before committing.", + "id": "US-26-05", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The wizard guides the user through sequential steps: About, Owner, Classification, Scheduling, Availability, and Target dataset; the user can continue to the next step or navigate back to a previous step.", + "id": "US-26-06", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "In the Target dataset step, the user can add schema attributes manually, import them from a file, or populate them from a Glue table.", + "id": "US-26-07", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-26", + "id": "US-26", + "state": "active", + "summary": { + "active_acs": 7, + "coverage_pct": 42.9, + "covered_acs": 3, + "total_acs": 7 + }, + "title": "Create Domain" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user can navigate to the \"Data flow\" tab on the Domain Detail page and the tab URL changes to /auth/all-domains/{domainId}/{version}/data-flow.", + "id": "US-30-01", + "state": "Planned", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The Data flow tab displays a visualisation of the domain's upstream data sources and downstream consumers.", + "id": "US-30-02", + "state": "Planned", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "When a domain has no configured data sources or consumers, the tab shows an empty-state message.", + "id": "US-30-03", + "state": "Planned", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-30", + "id": "US-30", + "state": "planned", + "summary": { + "active_acs": 0, + "coverage_pct": null, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "View Domain Data Flow" + }, + { + "acceptance_criteria": [], + "full_id": "absa-group/unify-living-documentation/US-7", + "id": "US-7", + "state": null, + "summary": { + "active_acs": 0, + "coverage_pct": null, + "covered_acs": 0, + "total_acs": 0 + }, + "title": "Delete Domain" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The domain name and country are locked for edit after domain creation.", + "id": "US-10-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can edit editable fields such as description, type of domain, target dataset, classification and schedule.", + "id": "US-10-02", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can add target data set schema: manually, import from file, populate glue table.", + "id": "US-10-03", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Invalid updates on cost center shows an error message on label.", + "id": "US-10-04", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can save changes — updated values are persisted and visible on reload.", + "id": "US-10-05", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can cancel edit — original values are restored without saving.", + "id": "US-10-06", + "state": "Active", + "version": "v1.8.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-10", + "id": "US-10", + "state": "active", + "summary": { + "active_acs": 6, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 6 + }, + "title": "Edit Domain" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user can navigate to the Hygiene page for a domain via the \"Survivorship\" option in the domain card More Actions menu on the Dashboard.", + "id": "US-31-01", + "state": "Planned", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The Hygiene page displays a metrics bar containing cards for: Coverage, Domain optimisation score, Total records from last run, Data quality score, and Consumers.", + "id": "US-31-02", + "state": "Planned", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each metric card displays the current computed value for the domain; when a metric is not yet available, the card shows an \"under development\" indicator.", + "id": "US-31-03", + "state": "Planned", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The Hygiene page also displays the standard domain About information below the metrics bar, consistent with the Domain Detail About tab.", + "id": "US-31-04", + "state": "Planned", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-31", + "id": "US-31", + "state": "planned", + "summary": { + "active_acs": 0, + "coverage_pct": null, + "covered_acs": 0, + "total_acs": 4 + }, + "title": "View Domain Hygiene (Survivorship)" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An authenticated user can open the Import Domain dialog from the Dashboard or from the All Domains page using the \"Import domain\" button.", + "id": "US-22-01", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The Import Domain dialog requires the user to provide a domain name and select a JSON file before the submit button becomes enabled.", + "id": "US-22-02", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "After a successful import the dialog closes and the newly imported domain is visible in the domain list.", + "id": "US-22-03", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user who submits an invalid or malformed JSON file is shown an error message and the dialog remains open for correction.", + "id": "US-22-04", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user can cancel the import at any point; no domain is created and the dialog closes.", + "id": "US-22-05", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-22", + "id": "US-22", + "state": "active", + "summary": { + "active_acs": 5, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 5 + }, + "title": "Import Domain" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can search domain by name or partial name.", + "id": "US-13-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each domain from search results displays its domain details card such as name, version, primary owner, creation date, and domain type.", + "id": "US-13-02", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user can filter the domain list by one or more status values: Active, Draft, Cold, and Testing; the list updates to show only domains matching the selected statuses.", + "id": "US-13-03", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "When the search term and applied filters produce no matching domains, the list displays an empty-state message.", + "id": "US-13-04", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-13", + "id": "US-13", + "state": "active", + "summary": { + "active_acs": 4, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 4 + }, + "title": "Search Domain" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A domain owner can navigate to the \"Version management\" tab on the Domain Detail page and see the current version and its lifecycle status.", + "id": "US-29-01", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A domain owner can create a new draft version of a domain from the Version management tab; the new version appears with status \"Draft\".", + "id": "US-29-02", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A domain owner can start the testing phase for a draft version; the version status changes to \"Testing\".", + "id": "US-29-03", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A domain owner can make an active version cold from the Version management tab; the version status changes to \"Cold\".", + "id": "US-29-04", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A domain owner can delete a version from the Version management tab after confirming a deletion prompt; the version no longer appears in the list.", + "id": "US-29-05", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user who is not the domain owner sees the Version management tab in read-only mode — lifecycle action buttons are not visible.", + "id": "US-29-06", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-29", + "id": "US-29", + "state": "active", + "summary": { + "active_acs": 6, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 6 + }, + "title": "Manage Domain Versions" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can access the domain details, schema, run history, data feed, data flow, statistics, question, access and version management.", + "id": "US-16-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Domain card details on dashboard displays information about name, creation date, primary owner attributes summary details about access request, questions and classification.", + "id": "US-16-02", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each Domain Detail tab (About, Schema, Run history, Data feed, Data flow, Statistics, Questions, Access, Version management) has its own distinct URL, allowing deep-linking directly to any tab.", + "id": "US-16-03", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The Schema tab displays the domain attribute list in a tree table; when no attributes are defined, an empty state is shown.", + "id": "US-16-04", + "state": "Active", + "version": "v1.9.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The More Actions menu on the Domain Detail page shows a different set of actions depending on ownership: domain owners see \"Edit domain\", \"Delete domain\", \"Grant access\", and \"Manage versions\" in addition to the options available to all users.", + "id": "US-16-05", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-16", + "id": "US-16", + "state": "active", + "summary": { + "active_acs": 5, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 5 + }, + "title": "View Domain" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when a domain is successfully created.", + "id": "US-19-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when a question is successfully created.", + "id": "US-19-02", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when domain is successfully modified.", + "id": "US-19-03", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when access to domain successfully requested.", + "id": "US-19-04", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when user successfully replied on a question.", + "id": "US-19-05", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when question is marked as helpful.", + "id": "US-19-06", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when question is unmarked as helpful.", + "id": "US-19-07", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when data feed successfully added.", + "id": "US-19-08", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when data feed edited successfully.", + "id": "US-19-09", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when a question is successfully deleted.", + "id": "US-19-10", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when a domain is successfully deleted.", + "id": "US-19-11", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when a data feed is successfully deleted.", + "id": "US-19-12", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when a domain access request is approved or rejected.", + "id": "US-19-13", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An on-screen notification is displayed when a domain is successfully imported.", + "id": "US-19-14", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-19", + "id": "US-19", + "state": "active", + "summary": { + "active_acs": 14, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 14 + }, + "title": "On-screen Notifications" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Only the owner can close one or more questions.", + "id": "US-4-01", + "state": "Active", + "version": "v1.2.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A closed question cannot receive new replies.", + "id": "US-4-02", + "state": "Active", + "version": "v1.2.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User cannot close an already-closed question.", + "id": "US-4-03", + "state": "Active", + "version": "v1.2.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-4", + "id": "US-4", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "Close Question" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The user clicks on tab question to access ask a question button to fill in all mandatory fields such as subject, description and select a domain for a new question.", + "id": "US-5-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The post question button is enabled when all mandatory fields are filled.", + "id": "US-5-02", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Question can hold any amount of Tags including no one.", + "id": "US-5-03", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A successfully submitted question is immediately visible in the question list.", + "id": "US-5-04", + "state": "Active", + "version": "v1.8.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-5", + "id": "US-5", + "state": "active", + "summary": { + "active_acs": 4, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 4 + }, + "title": "Create Question" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can delete question.", + "id": "US-8-01", + "state": "Active", + "version": "v1.2.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User is prompted with a confirmation dialog before question deletion.", + "id": "US-8-02", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A user who is not the question owner cannot delete the question.", + "id": "US-8-03", + "state": "Active", + "version": "v1.8.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-8", + "id": "US-8", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "Delete Question" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "The question owner can mark one reply on their question as helpful.", + "id": "US-20-01", + "state": "Active", + "version": "v1.2.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "A question can have at most one reply marked as helpful at a time.", + "id": "US-20-02", + "state": "Active", + "version": "v1.2.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-20", + "id": "US-20", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 2 + }, + "title": "Mark Question Helpful" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can reply one or more questions.", + "id": "US-11-01", + "state": "Active", + "version": "v1.2.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each question can have more than one dialogue reply.", + "id": "US-11-02", + "state": "Active", + "version": "v1.2.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-11", + "id": "US-11", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 2 + }, + "title": "Reply Question" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can search question by name or partial name.", + "id": "US-14-01", + "state": "Active", + "version": "v1.2.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Each question from search results displays its details such as subject, question details, domain name, creation date, and question owner.", + "id": "US-14-02", + "state": "Active", + "version": "v1.2.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-14", + "id": "US-14", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 2 + }, + "title": "Search Question" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can unmark one or more questions that marked as helpful.", + "id": "US-21-01", + "state": "Active", + "version": "v1.5.0" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "After unmarking, the reply no longer displays the helpful indicator.", + "id": "US-21-02", + "state": "Active", + "version": "v1.8.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Only the question owner can unmark a helpful reply.", + "id": "US-21-03", + "state": "Active", + "version": "v1.8.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-21", + "id": "US-21", + "state": "active", + "summary": { + "active_acs": 3, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 3 + }, + "title": "Unmark Helpful Question Reply" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "User can access the question details.", + "id": "US-17-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "Question details displays information such as subject, details, domain, creation date and the question owner.", + "id": "US-17-02", + "state": "Active", + "version": "v1.4.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-17", + "id": "US-17", + "state": "active", + "summary": { + "active_acs": 2, + "coverage_pct": 0.0, + "covered_acs": 0, + "total_acs": 2 + }, + "title": "View Question" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/user_login.feature/user-logs-in-with-valid-credentials", + "scenario_name": "User logs in with valid credentials", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "The login screen is accessible and displays all {aspects}. Aspects: username field, password field, login button", + "id": "US-1-01", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/user_login.feature/user-logs-in-with-valid-credentials", + "scenario_name": "User logs in with valid credentials", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "The user with valid username and password successfully logs in.", + "id": "US-1-02", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/user_login.feature/user-logs-in-with-invalid-credentials", + "scenario_name": "User logs in with invalid credentials", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "The user with invalid username and password is prompted with error message on screen.", + "id": "US-1-03", + "state": "Active", + "version": "v1.4.2" + }, + { + "coverage": { + "status": "not_covered", + "test_count": 0, + "tests": [] + }, + "description": "An authenticated user whose session has expired is redirected to the login screen with an informative message.", + "id": "US-1-04", + "state": "Active", + "version": "v2.2.7" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/user_login.feature/already-logged-in-user-navigating-to-login-url-is-redirected-to-dashboard", + "scenario_name": "Already logged-in user navigating to login URL is redirected to dashboard", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "A user who is already logged in and navigates to the login URL is automatically redirected to the dashboard.", + "id": "US-1-05", + "state": "Active", + "version": "v2.2.7" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/user_login.feature/user-logs-in-with-invalid-credentials", + "scenario_name": "User logs in with invalid credentials", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "The login button should be disabled until {aspects} fields are filled out. Aspects: username field, password field", + "id": "US-1-06", + "state": "Active", + "version": "v2.2.7" + }, + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/user_login.feature/user-initiates-msal-sso-login-and-is-redirected-to-the-corporate-identity-provid", + "scenario_name": "User initiates MSAL SSO login and is redirected to the corporate identity provider", + "source": { + "file": "user_login.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression" + ] + } + ] + }, + "description": "A user can initiate login via the MSAL SSO option on the login screen and is redirected through the corporate identity provider to complete authentication.", + "id": "US-1-07", + "state": "Active", + "version": "v1.9.0" + } + ], + "full_id": "absa-group/unify-living-documentation/US-1", + "id": "US-1", + "state": "active", + "summary": { + "active_acs": 7, + "coverage_pct": 85.7, + "covered_acs": 6, + "total_acs": 7 + }, + "title": "User Login" + }, + { + "acceptance_criteria": [ + { + "coverage": { + "status": "covered", + "test_count": 1, + "tests": [ + { + "id": "absa-group/unify-living-documentation/user_logout.feature/user-logs-out", + "scenario_name": "User logs out", + "source": { + "file": "user_logout.feature", + "org": "absa-group", + "repo": "unify-living-documentation" + }, + "tags": [ + "Regression", + "skip" + ] + } + ] + }, + "description": "The user can log out and is redirected to the login screen.", + "id": "US-15-01", + "state": "Active", + "version": "v1.4.2" + } + ], + "full_id": "absa-group/unify-living-documentation/US-15", + "id": "US-15", + "state": "active", + "summary": { + "active_acs": 1, + "coverage_pct": 100.0, + "covered_acs": 1, + "total_acs": 1 + }, + "title": "User Logout" + } + ] +} diff --git a/docs/architecture.md b/docs/architecture.md index e14a311..8fdc51e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,17 +51,16 @@ graph LR 1. **Collector Action** (`AbsaOSS/living-doc-collector-gh`) - Collects issues from GitHub - - Outputs: `doc-issues.json` + - Outputs: `doc-issues.json`, `doc-source.json`, `ui-tests.json` 2. **Adapter** (`packages/adapters/collector_gh`) - Detects producer format - Parses input into internal representation - Outputs: `AdapterResult` -3. **Service** (`packages/services/normalize_issues`) - - Normalizes markdown sections - - Builds canonical JSON structure - - Outputs: `pdf_ready.json` +3. **Services** (`packages/services/*`) + - `normalize_issues`: Normalizes markdown sections into PDF-ready JSON + - `coverage_matrix`: Cross-references User Stories with UI tests into an AC-level coverage matrix 4. **Dataset** (`packages/datasets_pdf`) - Validates output against schema @@ -140,6 +139,7 @@ graph TD Adapters --> CollectorGH[collector_gh/] Services --> NormalizeIssues[normalize_issues/] + Services --> CoverageMatrix[coverage_matrix/] Apps --> CLI[cli/] @@ -165,10 +165,18 @@ graph TD ServiceSrc --> Service[service.py] ServiceSrc --> Normalizer[normalizer.py] ServiceSrc --> Builder[builder.py] - + + CoverageMatrix --> CovSrc[src/living_doc_service_coverage_matrix/] + CovSrc --> CovService[service.py] + CovSrc --> CovLoader[loader.py] + CovSrc --> CovMatcher[matcher.py] + CovSrc --> CovSummary[summary.py] + CovSrc --> CovModel[model/coverage_item.py] + CLI --> CLISrc[src/living_doc_cli/] CLISrc --> Main[main.py] CLISrc --> Commands[commands/normalize_issues.py] + CLISrc --> CovCmd[commands/coverage_matrix.py] style Root fill:#e1f5ff,stroke:#0288d1 style Core fill:#fff9c4,stroke:#f57f17 @@ -183,15 +191,18 @@ graph TD ```mermaid graph TD CLI[apps/cli] --> NormalizeService[packages/services/normalize_issues] + CLI --> CoverageService[packages/services/coverage_matrix] NormalizeService --> Core[packages/core] NormalizeService --> Datasets[packages/datasets_pdf] NormalizeService --> CollectorGH[packages/adapters/collector_gh] - + CoverageService --> Core + CollectorGH --> Core Datasets --> Core - + style CLI fill:#fce4ec,stroke:#c2185b style NormalizeService fill:#ffe0b2,stroke:#e64a19 + style CoverageService fill:#ffe0b2,stroke:#e64a19 style Core fill:#fff9c4,stroke:#f57f17 style Datasets fill:#f3e5f5,stroke:#7b1fa2 style CollectorGH fill:#e8f5e9,stroke:#388e3c @@ -201,8 +212,8 @@ graph TD - **Core** has no dependencies (lowest level) - **Datasets** depends only on Core - **Adapters** depend on Core -- **Services** depend on Core, Datasets, and specific Adapters -- **CLI** depends on Core and specific Services +- **Services** depend on Core, and optionally Datasets and specific Adapters +- **CLI** depends on Core and all Services --- @@ -267,6 +278,54 @@ graph TB --- +### Coverage-Matrix Service Components + +```mermaid +graph TB + subgraph "Service Package (packages/services/coverage_matrix)" + CovService[service.py
Orchestration] + CovLoader[loader.py
I/O Boundary] + CovMatcher[matcher.py
Pure Matching Logic] + CovSummary[summary.py
Pure Tallying] + CovModel[model/coverage_item.py
Output Dataclasses] + end + + subgraph "Core Package (packages/core)" + JsonUtils2[json_utils.py] + LoggingConfig2[logging_config.py] + Errors2[errors.py] + end + + CovService --> CovLoader + CovService --> CovMatcher + CovMatcher --> CovSummary + CovMatcher --> CovModel + CovSummary --> CovModel + CovService --> JsonUtils2 + CovService --> LoggingConfig2 + CovService --> Errors2 + + style CovService fill:#ffe0b2,stroke:#e64a19,stroke-width:3px + style CovLoader fill:#e3f2fd,stroke:#1976d2 + style CovMatcher fill:#f3e5f5,stroke:#7b1fa2 + style CovSummary fill:#e8f5e9,stroke:#388e3c +``` + +**Component Responsibilities:** + +- **service.py**: Orchestration — loads inputs, calls matcher, writes output, enforces `--fail-under` +- **loader.py**: Pure I/O — reads `doc-source.json` (bare array or envelope) and `ui-tests.json` +- **matcher.py**: Pure transformation — builds the `CoverageMatrix` from parsed lists (no I/O) +- **summary.py**: Pure tallying — `compute_us_summary()` and `compute_summary()` with deprecated-AC exclusion +- **model/coverage_item.py**: Output dataclasses that serialize to `coverage-matrix.json` + +**Key design constraints:** +- `matcher.py` is a pure function: no I/O, no logging, fully testable without mocks +- Deprecated ACs are included in the matrix but excluded from `coverage_pct` computation +- Unresolved `us_id` values land in `unlinked_tests`; unknown `ac_id` references land in `stale_ac_refs` + +--- + ## Adapter Pattern ### Adapter Selection and Execution diff --git a/docs/contracts.md b/docs/contracts.md index 254417d..f2c3cb3 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -7,6 +7,7 @@ Quick reference for all external-facing contracts. Changes to items below requir - [CLI Interface](#cli-interface) - [Input Contract: `doc-issues.json`](#input-contract-doc-issuesjson) - [Output Contract: `pdf_ready.json`](#output-contract-pdf_readyjson) +- [Output Contract: `coverage-matrix.json`](#output-contract-coverage-matrixjson) - [Audit Envelope (v1.0)](#audit-envelope-v10) - [JSON Schemas](#json-schemas) - [Change Control](#change-control) @@ -16,7 +17,7 @@ Quick reference for all external-facing contracts. Changes to items below requir ## CLI Interface -**Command:** `living-doc normalize-issues` +### `living-doc normalize-issues` | Argument | Type | Required | Default | Description | |----------|------|----------|---------|-------------| @@ -27,7 +28,7 @@ Quick reference for all external-facing contracts. Changes to items below requir | `--document-version` | string | No | from input | Override `meta.document_version` | | `--verbose` | flag | No | `false` | Enable verbose logging | -### Exit Codes +### Exit Codes (`normalize-issues`) | Code | Condition | Error Prefix | |------|-----------|--------------| @@ -42,6 +43,25 @@ Error format: `{prefix} {detail}. {guidance}` --- +### `living-doc coverage-matrix` + +| Argument | Type | Required | Default | Description | +|----------|------|----------|---------|-------------| +| `--doc-input` | path | Yes | — | Path to US+AC doc JSON (`doc-source.json` / `doc-issues.json`) | +| `--tests-input` | path | Yes | — | Path to ui-tests JSON (`ui-tests.json`) | +| `--output` | path | Yes | — | Destination path for `coverage-matrix.json` | +| `--fail-under` | float | No | disabled | Exit code 1 if `coverage_pct < N` | +| `--verbose` | flag | No | `false` | Enable verbose logging | + +### Exit Codes (`coverage-matrix`) + +| Code | Condition | +|------|-----------| +| 0 | Success | +| 1 | Any error (invalid input, I/O failure, coverage below `--fail-under`) | + +--- + ## Input Contract: `doc-issues.json` Produced by [living-doc-collector-gh](https://github.com/AbsaOSS/living-doc-collector-gh). @@ -122,6 +142,44 @@ Issue body `##` headings map to canonical section keys (case-insensitive): --- +## Output Contract: `coverage-matrix.json` + +**Schema version:** `"coverage-matrix-v1.0.0"` (field `schema_version`) + +Produced by `living-doc coverage-matrix`. Consumed by downstream PDF / reporting generators. + +### Structure + +``` +coverage-matrix.json +├── schema_version: "coverage-matrix-v1.0.0" +├── generated_at: ISO-8601 timestamp +├── summary { total_user_stories, total_acs, active_acs, covered_acs, coverage_pct } +├── user_stories[] +│ ├── id, full_id, title, state +│ ├── summary { total_acs, active_acs, covered_acs, coverage_pct } +│ └── acceptance_criteria[] +│ ├── id, state, version, description +│ └── coverage { status, test_count, tests[] } +├── unlinked_tests[] ← scenarios with null or unresolved us_id +└── stale_ac_refs[] ← ac_ids that don't exist on the resolved US +``` + +### Coverage Status + +| Status | Condition | +|--------|-----------| +| `covered` | ≥1 scenario references this `ac_id` | +| `not_covered` | 0 scenarios reference this `ac_id` | + +### Coverage Percentage + +`coverage_pct = covered_active_acs / active_acs * 100` rounded to 1 dp. +Deprecated ACs (`state != "Active"`) are included in the matrix but **excluded from `coverage_pct`** so they cannot inflate scores. +`coverage_pct` is `null` when `active_acs == 0`. + +--- + ## Audit Envelope (v1.0) Lives at `meta.audit`. Preserves upstream provenance and tracks transformation steps. @@ -170,18 +228,22 @@ Each pipeline stage appends a trace entry: Machine-readable schemas are at: - `packages/datasets_pdf/schemas/pdf_ready_v1.schema.json` - `packages/datasets_pdf/schemas/audit_envelope_v1.schema.json` +- `packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/schema/coverage-matrix-v1.0.0-schema.json` -Pydantic models (source of truth): +Pydantic models (source of truth for PDF contracts): - `packages/datasets_pdf/src/living_doc_datasets_pdf/pdf_ready/v1/models.py` - `packages/datasets_pdf/src/living_doc_datasets_pdf/audit/v1/models.py` +Dataclasses (source of truth for coverage-matrix contract): +- `packages/services/coverage_matrix/src/living_doc_service_coverage_matrix/model/coverage_item.py` + --- ## Change Control ### Stable (breaking changes require major version bump) -- Schema field names, types, and meanings (v1.0) +- Schema field names, types, and meanings (`pdf_ready` v1.0, `coverage-matrix` v1.0.0) - `AdapterResult` model signature - CLI argument names and defaults - Exit codes and error message prefixes diff --git a/packages/services/coverage_matrix/tests/integration/test_golden_files.py b/packages/services/coverage_matrix/tests/integration/test_golden_files.py index be91a6e..caea573 100644 --- a/packages/services/coverage_matrix/tests/integration/test_golden_files.py +++ b/packages/services/coverage_matrix/tests/integration/test_golden_files.py @@ -73,4 +73,3 @@ def test_golden_summary_and_buckets(tmp_path): assert len(matrix.stale_ac_refs) == 1 assert matrix.stale_ac_refs[0].stale_ac_id == "US-1-99" assert matrix.stale_ac_refs[0].us_id == "US-1" -