run_conformance.py

#!/usr/bin/env python3
"""Run SDL's own automated test suite (test/testautomation*.c of the 3.4.18
release, 27 suites) against `import mcpplibs.mwrap.sdl3;`.

Upstream's suites are C against <SDL3/SDL.h>. Here each one is compiled as
C++ with the SDL headers replaced by the module, the way a program using the
module is written:

  - the suites' `#include <SDL3/...>` lines are removed and every file
    begins with the prelude: <sdl3m_side_macros.h>, then
    `import mcpplibs.mwrap.sdl3;`, `import sdl3m.test;` and
    `using namespace mcpplibs::mwrap::sdl3[::macros];`. So every SDL
    function, type and enumerator they use is the module's, every constant
    and computation macro the macros module's (SDL_INIT_VIDEO, SDLK_*,
    SDL_PIXELFORMAT_*, SDL_BITSPERPIXEL, SDL_zero ...), and every inline
    function an adaptation (SDL_RectEmpty, SDL_Swap32 ...).
  - sdl3m_side_macros.h holds only the macros the module does not map — what
    <mwrap/sdl3/macros.hpp> is for: the assertions, SDL_COMPILE_TIME_ASSERT,
    SDL_arraysize-free tests of SDL_PLATFORM_*, attributes, SDL 2's names and
    the macros also used by the preprocessor (policy.PREPROCESSOR, the printf
    formats SDL_PRIs64 &c., concatenated with literals). It is generated on
    this target, from the compiler's `-E -dD` of the headers, so it holds
    this target's definitions and no declaration at all.
  - patches/testautomation-cxx.patch makes upstream's C valid C++ (casts from
    void *, enums from integers; the patch lists each kind).
  - `#include "../src/joystick/usb_ids.h"` becomes "joystick/usb_ids.h"
    (compat.sdl3's source directory is on the include path).
    <SDL3/SDL_intrin.h> and SDL's "SDL_build_config.h" stay #includes: the
    module does not wrap them (tools/inventory.py, NOT_WRAPPED).

The test framework the suites drive (SDL3_test: src/test/*.c, the harness,
fuzzer, assertions) is compiled as C against the headers, as upstream builds
it; the suites reach it through `sdl3m.test`, a module generated from
<SDL3/SDL_test.h> the way the wrapper itself is. The runner is upstream's
testautomation.c, transformed the same way.

It runs as upstream's CTest runs it (test/CMakeLists.txt): in a directory
holding the test resources, with SDL_VIDEO_DRIVER=dummy,
SDL_AUDIO_DRIVER=dummy and SDL_ASSERT=abort, and a second time with
SDL_CPU_FEATURE_MASK=-all (testautomation-no-simd). Every suite must pass.
Everything is generated in target/conformance; run `mcpp build` in the
repository first (the side macros come from its compile_commands.json).
"""
from __future__ import annotations

import argparse
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys

sys.path.insert(0, str(Path(__file__).resolve().parent))
from inventory import Group, declarations, fetch, read_header, surface  # noqa: E402
import check_exports  # noqa: E402
import gen_exports  # noqa: E402
import policy  # noqa: E402

PRELUDE = '''// Generated by tools/run_conformance.py from SDL 3.4.18's {name}: upstream's
// test against `import mcpplibs.mwrap.sdl3;` (patches/testautomation-cxx.patch
// applied). The SDL headers are not included.
#include "sdl3m_side_macros.h"
import mcpplibs.mwrap.sdl3;
import sdl3m.test;
using namespace mcpplibs::mwrap::sdl3;
using namespace mcpplibs::mwrap::sdl3::macros;
'''

SYSTEM_HEADERS = '''// The C library headers SDL's headers include for a program.
#include <stdarg.h>
#include <stddef.h>
#include <stdint.h>
#include <inttypes.h>
#include <string.h>
#include <wchar.h>
#include <stdlib.h>
#include <stdio.h>
#include <signal.h>
#if defined(_MSC_VER)
#include <malloc.h>
#include <intrin.h>
#elif defined(__has_include)
#if __has_include(<alloca.h>)
#include <alloca.h>
#endif
#endif
'''

MANIFEST = '''# Generated by tools/run_conformance.py: SDL's testautomation against the module.
[package]
name        = "testautomation"
version     = "3.4.18"
description = "SDL 3.4.18's automated test suite, compiled against import mcpplibs.mwrap.sdl3"
license     = "Zlib"

[targets.testautomation]
kind = "bin"
main = "src/testautomation.cpp"

[build]
sources      = ["src/*.cppm", "src/suites/*.cpp", "src/sdltest/*.c"]
include_dirs = ["src", "src/sdltest"]

[dependencies]
"mcpplibs.mwrap.sdl3" = {{ path = {root} }}
'''


def write(path: Path, text: str):
    """Write only when different: unchanged inputs keep their timestamps."""
    path.parent.mkdir(parents=True, exist_ok=True)
    if not path.exists() or path.read_text(encoding='utf-8') != text:
        path.write_text(text, encoding='utf-8', newline='\n')


def patched_tests(root: Path, upstream: Path) -> Path:
    stage = root / 'target/conformance/upstream'
    if stage.exists():
        shutil.rmtree(stage)
    (stage / 'test').mkdir(parents=True)
    for f in (upstream / 'test').glob('testautomation*'):
        shutil.copy2(f, stage / 'test' / f.name)
    subprocess.run(['git', 'apply', '-p1', str(root / 'patches/testautomation-cxx.patch')], cwd=stage, check=True)
    return stage / 'test'


def transform(text: str, name: str, prelude: bool) -> str:
    out = []
    for line in text.splitlines():
        if re.match(r'\s*#\s*include\s*<SDL3/SDL(_main|_test\w*)?\.h>', line):
            continue
        line = line.replace('"../src/', '"')
        out.append(line)
    body = '\n'.join(out) + '\n'
    return (PRELUDE.format(name=name) + body) if prelude else body


def test_module(upstream: Path) -> str:
    """sdl3m.test: SDL_test.h's declarations, exported as the wrapper exports SDL's."""
    include = upstream / 'include/SDL3'
    groups: dict[int, Group] = {}
    names = []
    for h in sorted(include.glob('SDL_test*.h')):
        header = read_header(h, groups)
        for d, _, terms in header.decls:
            if d.kind in ('function', 'type', 'tag', 'enumerator') and d.name not in names:
                if terms:
                    sys.exit(f'{h.name}: {d.name} is conditional; sdl3m.test does not handle that')
                names.append(d.name)
        # A few are declared without SDLCALL (Uint64 SDLTest_RandomUint64(void);).
        for body, _ in header.unknown:
            m = re.match(r'[\w\s\*]*?\b(SDLTest_\w+)\s*\(', body) or re.match(r'extern\s+[\w\s\*]+?\b(\w+)\s*;$', body)
            if not m:
                sys.exit(f'{h.name}: a statement sdl3m.test does not understand: {" ".join(body.split())[:100]}')
            if m.group(1) not in names:
                names.append(m.group(1))
    lines = ['// Generated by tools/run_conformance.py: SDL_test.h (the SDL3_test framework the',
             "// suites drive) as a module, exported the way the wrapper exports SDL's.",
             'module;', '#include <SDL3/SDL_test.h>', 'export module sdl3m.test;', '']
    lines += [f'export using ::{n};' for n in names]
    return '\n'.join(lines) + '\n'


def side_macros(root: Path) -> str:
    """The macros the module does not map, as this target defines them."""
    s = surface(fetch(root))
    classes = {n: k for n, _, k, _ in gen_exports.macro_table(s)}
    cmd = check_exports.compile_command(root)
    probe = gen_exports.INCLUDES + '#include <SDL3/SDL_test.h>\n'
    text = check_exports.preprocess(cmd, probe, root / 'target/conformance/probe', 'side.cpp', True)
    defs: dict[str, str] = {}
    current_sdl = False
    for line in text.splitlines():
        m = re.match(r'#\s*(?:line\s+)?\d+\s+"([^"]*)"', line)
        if m:
            current_sdl = '/SDL3/' in m.group(1).replace('\\\\', '/').replace('\\', '/')
            continue
        d = re.match(r'#\s*(define|undef)\s+(\w+)', line)
        if not d:
            continue
        if d.group(1) == 'undef':
            defs.pop(d.group(2), None)
        elif current_sdl:
            defs.pop(d.group(2), None)
            defs[d.group(2)] = line
    declared = set(s.entries) | set(policy.ADAPTED)
    out = ['// Generated by tools/run_conformance.py on this target: the macros of SDL\'s',
           '// headers that mcpplibs.mwrap.sdl3 does not map (and SDL_test.h\'s), from the',
           "// compiler's -E -dD. No declaration: those come from the modules.",
           '#pragma once', SYSTEM_HEADERS, '#define SDL_MAIN_HANDLED 1']
    kept = 0
    for name, line in defs.items():
        k = classes.get(name)
        mapped = k in ('constant', 'string', 'function')
        if (mapped and name not in policy.PREPROCESSOR) or name in declared:
            continue
        out.append(line)
        kept += 1
    print(f'side macros: {kept} of {len(defs)} kept as macros; the rest are the module\'s', flush=True)
    return '\n'.join(out) + '\n'


def build(project: Path, mcpp: str, toolchain: str | None, profile: str) -> Path:
    command = [mcpp, 'build', '--profile', profile] + (['--toolchain', toolchain] if toolchain else [])
    print('+', ' '.join(command), f'(in {project})', flush=True)
    subprocess.run(command, cwd=project, check=True)
    name = 'testautomation.exe' if os.name == 'nt' else 'testautomation'
    found = sorted((p for p in (project / 'target').rglob(name) if p.parent.name == 'bin'),
                   key=lambda p: p.stat().st_mtime)
    if not found:
        sys.exit('testautomation was not built')
    return found[-1]


def run(binary: Path, rundir: Path, extra_env: dict, label: str, timeout: int) -> tuple[bool, str]:
    env = dict(os.environ, SDL_VIDEO_DRIVER='dummy', SDL_AUDIO_DRIVER='dummy', SDL_ASSERT='abort', **extra_env)
    print(f'+ {binary.name}  ({label}; in {rundir})', flush=True)
    try:
        proc = subprocess.run([str(binary)], cwd=rundir, env=env, capture_output=True, text=True,
                              encoding='utf-8', errors='replace', timeout=timeout)
    except subprocess.TimeoutExpired as e:
        return False, f'timed out after {timeout} s\n' + ((e.stdout or '') if isinstance(e.stdout, str) else '')
    output = proc.stdout + proc.stderr
    return proc.returncode == 0, output


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--mcpp', default='mcpp')
    parser.add_argument('--toolchain')
    parser.add_argument('--profile', default='dev')
    parser.add_argument('--timeout', type=int, default=600)
    args = parser.parse_args()
    root = Path(__file__).resolve().parents[1]
    upstream = fetch(root)
    project = root / 'target/conformance/project'
    tests = patched_tests(root, upstream)

    write(project / 'mcpp.toml', MANIFEST.format(root=json.dumps(root.as_posix())))
    src = project / 'src'
    write(src / 'sdl3m_side_macros.h', side_macros(root))
    write(src / 'sdl3m_test.cppm', test_module(upstream))
    for f in sorted(tests.glob('testautomation*.h')):
        write(src / f.name, transform(f.read_text(encoding='utf-8'), f.name, prelude=False))
    for f in sorted(tests.glob('testautomation*.c')):
        dest = src / ('testautomation.cpp' if f.name == 'testautomation.c' else f'suites/{f.stem}.cpp')
        write(dest, transform(f.read_text(encoding='utf-8'), f.name, prelude=True))
    for f in (upstream / 'src/test').iterdir():
        if f.suffix in ('.c', '.h'):
            write(src / 'sdltest' / f.name, f.read_text(encoding='utf-8'))

    binary = build(project, args.mcpp, args.toolchain, args.profile)
    rundir = root / 'target/conformance/run'
    if rundir.exists():
        shutil.rmtree(rundir)
    rundir.mkdir(parents=True)
    for pattern in ('*.png', '*.wav', '*.csv', '*.hex', 'moose.dat', 'utf8.txt'):  # test/CMakeLists.txt RESOURCE_FILES
        for f in (upstream / 'test').glob(pattern):
            shutil.copy2(f, rundir / f.name)

    failures = []
    for label, env in (('testautomation', {}), ('testautomation-no-simd', {'SDL_CPU_FEATURE_MASK': '-all'})):
        ok, output = run(binary, rundir, env, label, args.timeout)
        output = re.sub(r'\x1b\[[0-9;]*m', '', output)
        lines = output.splitlines()
        suites = [l.split(': ', 1)[-1] for l in lines if re.search(r">>> Suite '.*': ", l)]
        skipped = [l.split(': ', 1)[-1] for l in lines if re.search(r">>> Test '.*': Skipped", l)]
        total = [l.split(': ', 1)[-1] for l in lines if 'Run Summary:' in l]
        print(f'{label}: {len(suites)} suites; ' + (total[-1] if total else 'no run summary'), flush=True)
        for line in suites + skipped:
            print('  ' + line.strip(), flush=True)
        if not ok or not total or 'Failed=0' not in total[-1]:
            failed = [l for l in lines if re.search(r'Failed|Assert .*failed|Assertion failure', l, flags=re.I)]
            print('\n'.join(failed[-80:]) or output[-6000:], flush=True)
            failures.append(label)
    if failures:
        sys.exit('failed: ' + ', '.join(failures))
    print('SDL 3.4.18 testautomation: every suite passed against the module, with and without SIMD')


if __name__ == '__main__':
    main()