Improve test automation

tests/images/test_image_summary_report.py

@@ -0,0 +1,204 @@
+import runpy
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+import pytest
+
+from tests.integration.image_summary_report import main, render_failing_image_summary
+
+
+def test_render_failing_image_summary_uses_header_and_exact_task_tokens(
+    tmp_path: Path,
+) -> None:
+    summary: Path = tmp_path / "test_images_summary.md"
+    summary.write_text(
+        "\n".join(
+            [
+                "# Summary of test results",
+                "",
+                "| Test name | Missing images | Total images | Needs review | Correct images | Identical | Cosmetic only | Severity |",
+                "| --- | --- | --- | --- | --- | --- | --- | --- |",
+                "| /plots/global_time_series/case_a | 1 | 10 | 0 | 9 | 9 | 0 | medium |",
+                "| /plots/notglobal_time_seriesx/case_b | 0 | 10 | 1 | 9 | 9 | 0 | low |",
+                "| /plots/e3sm_diags/case_c | 0 | 10 | 0 | 10 | 10 | 0 | none |",
+            ]
+        )
+        + "\n"
+    )
+
+    tasks: List[str] = ["e3sm_diags", "global_time_series"]
+    report: str = render_failing_image_summary(str(summary), tasks)
+
+    assert "`global_time_series`" in report
+    assert "`other`" in report
+    assert "/plots/global_time_series/case_a" in report
+    assert "/plots/notglobal_time_seriesx/case_b" in report
+    assert "/plots/e3sm_diags/case_c" not in report
+    assert report.index("`global_time_series`") < report.index("`other`")
+
+
+def test_render_failing_image_summary_matches_cfg_task_suffixes(
+    tmp_path: Path,
+) -> None:
+    summary: Path = tmp_path / "test_images_summary.md"
+    summary.write_text(
+        "\n".join(
+            [
+                "# Summary of test results",
+                "",
+                "| Test name | Total images | Correct images | Identical | Cosmetic only | Missing images | Needs review | Severity |",
+                "| --- | --- | --- | --- | --- | --- | --- | --- |",
+                "| comprehensive_v2_e3sm_diags | 10 | 9 | 9 | 0 | 1 | 0 | medium |",
+                "| comprehensive_v2_global_time_series | 10 | 9 | 9 | 0 | 0 | 1 | low |",
+            ]
+        )
+        + "\n"
+    )
+
+    report: str = render_failing_image_summary(
+        str(summary), ["e3sm_diags", "global_time_series"]
+    )
+
+    assert "`e3sm_diags`" in report
+    assert "`global_time_series`" in report
+    assert "comprehensive_v2_e3sm_diags" in report
+    assert "comprehensive_v2_global_time_series" in report
+
+
+def test_render_failing_image_summary_reports_missing_columns(
+    tmp_path: Path,
+) -> None:
+    summary: Path = tmp_path / "test_images_summary.md"
+    summary.write_text(
+        "\n".join(
+            [
+                "# Summary of test results",
+                "",
+                "| Test name | Total images | Correct images | Severity |",
+                "| --- | --- | --- | --- |",
+                "| /plots/global_time_series/case_a | 10 | 9 | medium |",
+            ]
+        )
+        + "\n"
+    )
+
+    report: str = render_failing_image_summary(str(summary), ["global_time_series"])
+
+    assert report == "Unable to identify failing image-check columns.\n"
+
+
+def test_main_writes_report_to_stdout(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    summary: Path = tmp_path / "test_images_summary.md"
+    summary.write_text(
+        "\n".join(
+            [
+                "# Summary of test results",
+                "",
+                "| Test name | Total images | Correct images | Identical | Cosmetic only | Missing images | Needs review | Severity |",
+                "| --- | --- | --- | --- | --- | --- | --- | --- |",
+                "| /plots/global_time_series/case_a | 10 | 9 | 9 | 0 | 1 | 0 | medium |",
+            ]
+        )
+        + "\n"
+    )
+
+    exit_code: int = main([str(summary), "global_time_series"])
+    captured = capsys.readouterr()
+
+    assert exit_code == 0
+    assert captured.out == (
+        "`global_time_series`\n\n"
+        "| Test name | Total images | Correct images | Identical |"
+        " Cosmetic only | Missing images | Needs review | Severity |\n"
+        "| --- | --- | --- | --- | --- | --- | --- | --- |\n"
+        "| /plots/global_time_series/case_a | 10 | 9 | 9 | 0 | 1 | 0 |"
+        " medium |\n"
+    )
+
+
+def test_module_entry_point_writes_report_to_stdout(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    summary: Path = tmp_path / "test_images_summary.md"
+    summary.write_text(
+        "\n".join(
+            [
+                "# Summary of test results",
+                "",
+                "| Test name | Total images | Correct images | Identical | Cosmetic only | Missing images | Needs review | Severity |",
+                "| --- | --- | --- | --- | --- | --- | --- | --- |",
+                "| /plots/global_time_series/case_a | 10 | 9 | 9 | 0 | 1 | 0 | medium |",
+            ]
+        )
+        + "\n"
+    )
+
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "python",
+            str(summary),
+            "global_time_series",
+        ],
+    )
+
+    with pytest.raises(SystemExit) as excinfo:
+        runpy.run_module("tests.integration.image_summary_report", run_name="__main__")
+
+    captured = capsys.readouterr()
+    assert excinfo.value.code == 0
+    assert captured.out == (
+        "`global_time_series`\n\n"
+        "| Test name | Total images | Correct images | Identical |"
+        " Cosmetic only | Missing images | Needs review | Severity |\n"
+        "| --- | --- | --- | --- | --- | --- | --- | --- |\n"
+        "| /plots/global_time_series/case_a | 10 | 9 | 9 | 0 | 1 | 0 |"
+        " medium |\n"
+    )
+
+
+def test_shell_invocation_appends_failing_summary_to_report(tmp_path: Path) -> None:
+    summary: Path = tmp_path / "test_images_summary.md"
+    report_file: Path = tmp_path / "report.md"
+    repo_root: Path = Path(__file__).resolve().parents[2]
+    summary.write_text(
+        "\n".join(
+            [
+                "# Summary of test results",
+                "",
+                "| Test name | Total images | Correct images | Identical | Cosmetic only | Missing images | Needs review | Severity |",
+                "| --- | --- | --- | --- | --- | --- | --- | --- |",
+                "| /plots/global_time_series/case_a | 10 | 9 | 9 | 0 | 1 | 0 | medium |",
+            ]
+        )
+        + "\n"
+    )
+
+    subprocess.run(
+        [
+            "bash",
+            "-lc",
+            (
+                f'"{sys.executable}" -m tests.integration.image_summary_report '
+                f'"{summary}" global_time_series >> "{report_file}"'
+            ),
+        ],
+        check=True,
+        cwd=repo_root,
+    )
+
+    assert report_file.read_text() == (
+        "`global_time_series`\n\n"
+        "| Test name | Total images | Correct images | Identical |"
+        " Cosmetic only | Missing images | Needs review | Severity |\n"
+        "| --- | --- | --- | --- | --- | --- | --- | --- |\n"
+        "| /plots/global_time_series/case_a | 10 | 9 | 9 | 0 | 1 | 0 |"
+        " medium |\n"
+    )

tests/integration/image_summary_report.py

@@ -0,0 +1,110 @@
+"""Helpers for rendering the failing-only image-summary report section."""
+
+from __future__ import annotations
+
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import DefaultDict, List, Sequence, Tuple
+
+TableRow = Tuple[str, List[str]]
+
+
+def _parse_summary_table(
+    lines: Sequence[str],
+) -> Tuple[str | None, List[str], List[TableRow]]:
+    header: str | None = None
+    header_cols: List[str] = []
+    rows: List[TableRow] = []
+    in_table: bool = False
+
+    for line in lines:
+        stripped: str = line.strip()
+        if stripped.startswith("| Test name"):
+            header = line.rstrip("\n")
+            header_cols = [col.strip() for col in stripped.strip("|").split("|")]
+            in_table = True
+            continue
+        if in_table and stripped.startswith("| ---"):
+            continue
+        if in_table and stripped.startswith("|"):
+            cols: List[str] = [col.strip() for col in stripped.strip("|").split("|")]
+            rows.append((line.rstrip("\n"), cols))
+        elif in_table and not stripped.startswith("|"):
+            in_table = False
+
+    return header, header_cols, rows
+
+
+def _has_failures(value: str) -> bool:
+    digits: str = ""
+    for char in value:
+        if char.isdigit():
+            digits += char
+        else:
+            break
+    return digits not in ("", "0")
+
+
+def _match_task(name: str, tasks: Sequence[str]) -> str:
+    tokens = {token for token in re.split(r"[^A-Za-z0-9_]+", name) if token}
+    for task in tasks:
+        if name == task or name.endswith(f"_{task}") or task in tokens:
+            return task
+    return "other"
+
+
+def render_failing_image_summary(summary_file: str, tasks: Sequence[str]) -> str:
+    """Render the failing-only summary table grouped by task."""
+
+    lines: List[str] = Path(summary_file).read_text().splitlines()
+    header, header_cols, rows = _parse_summary_table(lines)
+
+    if (
+        header is None
+        or "Missing images" not in header_cols
+        or "Needs review" not in header_cols
+    ):
+        return "Unable to identify failing image-check columns.\n"
+
+    missing_idx: int = header_cols.index("Missing images")
+    needs_review_idx: int = header_cols.index("Needs review")
+    separator: str = "| " + " | ".join(["---"] * len(header_cols)) + " |"
+
+    failing: List[Tuple[str, str]] = []
+    for line, cols in rows:
+        if len(cols) <= max(missing_idx, needs_review_idx):
+            continue
+        if _has_failures(cols[missing_idx]) or _has_failures(cols[needs_review_idx]):
+            failing.append((line, cols[0]))
+
+    if not failing:
+        return "No failing image-check tests.\n"
+
+    by_task: DefaultDict[str, List[str]] = defaultdict(list)
+    for line, name in failing:
+        by_task[_match_task(name, tasks)].append(line)
+
+    output_lines: List[str] = []
+    for task in sorted(by_task):
+        output_lines.extend([f"`{task}`", "", header, separator, *by_task[task], ""])
+
+    return "\n".join(output_lines).rstrip() + "\n"
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args: Sequence[str] = sys.argv[1:] if argv is None else argv
+    if not args:
+        raise SystemExit(
+            "Usage: python -m tests.integration.image_summary_report <summary_file> [task...]"
+        )
+
+    summary_file: str = args[0]
+    tasks: Sequence[str] = args[1:]
+    sys.stdout.write(render_failing_image_summary(summary_file, tasks))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/integration/template_update_weekly_expected_files.sh

@@ -6,27 +6,27 @@
 # NOTE: in `tests` below, do *not* include the `zppy_weekly_` prefix, as that is added later.
 
 # Update all
-tests=("comprehensive_v2" "comprehensive_v3" "bundles" "legacy_3.1.0_comprehensive_v2" "legacy_3.1.0_comprehensive_v3" "legacy_3.1.0_bundles" "legacy_3.0.0_comprehensive_v2" "legacy_3.0.0_comprehensive_v3" "legacy_3.0.0_bundles")
+tests=("comprehensive_v2" "comprehensive_v3" "bundles" "legacy_3.1.0_comprehensive_v3" "legacy_3.0.0_comprehensive_v3")
 
 # Update regular only
 #tests=("comprehensive_v2" "comprehensive_v3" "bundles")
 
 # Update legacy 3.1.0 only
-#tests=("legacy_3.1.0_comprehensive_v2" "legacy_3.1.0_comprehensive_v3" "legacy_3.1.0_bundles")
+#tests=("legacy_3.1.0_comprehensive_v3")
 
 # Update legacy 3.0.0 only
-#tests=("legacy_3.0.0_comprehensive_v2" "legacy_3.0.0_comprehensive_v3" "legacy_3.0.0_bundles")
+#tests=("legacy_3.0.0_comprehensive_v3")
 
 for test_name in "${tests[@]}"
 do
     # Example for Chrysalis:
     #
     # expected_dir = /lcrc/group/e3sm/public_html/zppy_test_resources/
     #
-    # There are 9 subdirectories relevant to image checking:
+    # There are 5 subdirectories relevant to image checking:
     # 1-3. expected_bundles, expected_comprehensive_v2, expected_comprehensive_v3
-    # 4-6. expected_legacy_3.1.0_bundles, expected_legacy_3.1.0_comprehensive_v2, expected_legacy_3.1.0_comprehensive_v3
-    # 7-9. expected_legacy_3.0.0_bundles, expected_legacy_3.0.0_comprehensive_v2, expected_legacy_3.0.0_comprehensive_v3
+    # 4. expected_legacy_3.1.0_comprehensive_v3
+    # 5. expected_legacy_3.0.0_comprehensive_v3
     # Notice the subdirectories do *not* include the `zppy_weekly` prefix.
     #
     # Each of those subdirectories has a corresponding image list of the form:

tests/integration/template_weekly_bundles.cfg

@@ -79,7 +79,7 @@ years = "1985:1989:2",
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
-  years = "1985:1995:5",
+  years = "1985:1994:5","1995-1995",
 
   [[ land_monthly ]]
   active = #expand active_ts_month_lnd#

tests/integration/template_weekly_comprehensive_v3.cfg

@@ -139,7 +139,7 @@ walltime = "00:30:00"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
-  years = "1985:1995:5",
+  years = "1985:1994:5","1995-1995",
 
   [[ lnd_monthly_glb ]]
   active = #expand active_ts_month_lnd_glb#
@@ -148,7 +148,7 @@ walltime = "00:30:00"
   input_subdir = "archive/lnd/hist"
   mapping_file = "glb"
   vars = ""
-  years = "1985:1995:5",
+  years = "1985:1994:5","1995-1995",
 
   [[ land_monthly ]]
   active = #expand active_ts_month_lnd#