diff --git a/.github/workflows/build_matrix.yml b/.github/workflows/build_matrix.yml index f60f27d99c..ee92faf514 100644 --- a/.github/workflows/build_matrix.yml +++ b/.github/workflows/build_matrix.yml @@ -48,6 +48,10 @@ jobs: - name: "SCM Checkout" uses: actions/checkout@v4 + - name: "Runner info" + shell: bash + run: bash ci/runner_info.sh + - name: "Exclude workspace from Windows Defender" if: runner.os == 'Windows' shell: pwsh @@ -175,6 +179,16 @@ jobs: echo "SCCACHE_DIR=${{ runner.temp }}/sccache" >> $GITHUB_ENV echo "SCCACHE_GHA_ENABLED=false" >> $GITHUB_ENV echo "ZERO_AR_DATE=1" >> $GITHUB_ENV + # SCCACHE_CACHE_SIZE bounds the slot: the default is 10G, and a slot grows to its bound + # with objects of commits long gone. Unbounded slots fill the repository's cache quota, + # and the slots the nightly cells save once a day are the ones evicted, before the next + # nightly reads them. A windows /Z7 object is ~5x a posix one (debug info embedded in + # every .obj), hence its own bound. + if [ "$RUNNER_OS" = "Windows" ]; then + echo "SCCACHE_CACHE_SIZE=5000M" >> $GITHUB_ENV + else + echo "SCCACHE_CACHE_SIZE=2000M" >> $GITHUB_ENV + fi # Non-Windows: export the launcher via env (also wraps the GLFW/libhv # ExternalProject sub-builds — harmless on gcc/clang). Windows must NOT # export it: CMake seeds CMAKE__COMPILER_LAUNCHER from the env @@ -192,8 +206,9 @@ jobs: # no-op, so we delete first). PRs read-only. Bounded footprint: N configs × # one tarball, no run_id pileup. Every run restores, the cron included: the # nightly cells are the slow ones, and the cron's save is the seeder every PR - # lane warms from. A sanitizer cell's three phase jobs share one - # slot; only the tests phase writes it, so there is one writer per key. + # lane warms from. A sanitizer cell's tests and backend jobs share one + # slot the tests job writes; the slow job reads and writes a slot of its own, + # since the full test_aot it compiles is in no other job's objects. One writer per key. - name: "Restore sccache objects" uses: actions/cache/restore@v4 with: @@ -201,8 +216,8 @@ jobs: ${{ runner.temp }}/sccache .cache/daslang/test_cache .cache/daslang/_anon_* - key: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}@${{ github.run_id }} - restore-keys: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}@ + key: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}${{ matrix.phase == 'slow' && '-slow' || '' }}@${{ github.run_id }} + restore-keys: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}${{ matrix.phase == 'slow' && '-slow' || '' }}@ - name: "Override das_llvm_disabled for LLVM-free matrix cells" if: (matrix.target == 'windows' && matrix.architecture == 32) || matrix.llvm_disabled == 'ON' @@ -227,6 +242,13 @@ jobs: cmake --build ./build --config ${{ matrix.cmake_preset }} --target $lane --parallel \ || cmake --build ./build --config ${{ matrix.cmake_preset }} --target ${lane}_isolated --parallel + # windows 64 Debug: the default build and nothing else. windows Release and the posix Debug + # cells run the sweeps; what only this cell says is that the tree compiles and links as + # MSVC Debug. + - name: "Compile" + if: matrix.phase == 'compile' + run: cmake --build ./build --config ${{ matrix.cmake_preset }} --parallel + - name: "Nightly Tests" # Full AOT suite: builds the full test_aot (EXCLUDE_FROM_ALL, ~1080 AOT # TUs — the single biggest CI cost) via the run_tests_aot dependency and @@ -254,19 +276,19 @@ jobs: # The lane targets above are what compiles (Build only configures), so the slot is saved # here, and on a red sweep too: the objects a failed run compiled are as good as any. - name: "Save sccache objects" - if: ${{ !cancelled() && github.ref == 'refs/heads/master' && matrix.phase != 'slow' && matrix.phase != 'backend' }} + if: ${{ !cancelled() && github.ref == 'refs/heads/master' && matrix.phase != 'backend' }} uses: actions/cache/save@v4 with: path: | ${{ runner.temp }}/sccache .cache/daslang/test_cache .cache/daslang/_anon_* - key: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}@${{ github.run_id }} + key: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}${{ matrix.phase == 'slow' && '-slow' || '' }}@${{ github.run_id }} - name: "Prune older sccache slots" - if: ${{ !cancelled() && github.ref == 'refs/heads/master' && matrix.phase != 'slow' && matrix.phase != 'backend' }} + if: ${{ !cancelled() && github.ref == 'refs/heads/master' && matrix.phase != 'backend' }} env: GH_TOKEN: ${{ github.token }} - SCKEY: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}@ + SCKEY: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }}${{ matrix.fast_math == 'ON' && '-fastmath' || '' }}${{ matrix.phase == 'slow' && '-slow' || '' }}@ run: | # best effort, and the newest entry survives whatever this run saved: a save that produced # nothing must not wipe the last good slot, and a listing hiccup must not fail the lane @@ -275,3 +297,8 @@ jobs: --jq 'sort_by(.createdAt) | reverse | .[1:] | .[].id' \ | while read -r id; do gh cache delete "$id" -R ${{ github.repository }}; done true + + - name: "Runner info at the end" + if: ${{ !cancelled() }} + shell: bash + run: bash ci/runner_info.sh diff --git a/.github/workflows/extended_checks.yml b/.github/workflows/extended_checks.yml index 95ace4ea96..9af82401e9 100644 --- a/.github/workflows/extended_checks.yml +++ b/.github/workflows/extended_checks.yml @@ -78,6 +78,10 @@ jobs: # Full history so the lint step can `git diff` against the PR base branch. fetch-depth: 0 + - name: "Runner info" + shell: bash + run: bash ci/runner_info.sh + - name: "Exclude workspace from Windows Defender" if: runner.os == 'Windows' shell: pwsh @@ -112,12 +116,6 @@ jobs: run: | set -eux case "${{ matrix.target }}" in - darwin15) - brew install bison openssl - echo 'export PATH="/usr/local/opt/bison/bin:$PATH"' >> ~/.bash_profile - export LDFLAGS="-L/usr/local/opt/bison/lib -L$(brew --prefix openssl)/lib" - export CPPFLAGS="-I$(brew --prefix openssl)/include" - ;; linux) echo "MARCH=64" >> $GITHUB_ENV # apt.llvm.org carries LLVM 22 per Ubuntu codename; ubuntu-latest moves between them. @@ -294,6 +292,11 @@ jobs: | while read -r id; do gh cache delete "$id" -R ${{ github.repository }}; done true + - name: "Runner info at the end" + if: ${{ !cancelled() }} + shell: bash + run: bash ci/runner_info.sh + commit_messages: if: github.event_name == 'pull_request' && inputs.lane != 'extended_nightly' runs-on: ubuntu-latest diff --git a/ci/CMakeLists.txt b/ci/CMakeLists.txt index 1a19e9a7f6..2772143f5b 100644 --- a/ci/CMakeLists.txt +++ b/ci/CMakeLists.txt @@ -120,8 +120,14 @@ das_lane_group(run_extended_modules "Running the extended checks (modules)" ${DA das_lane_group(run_extended_all "Running every extended check" ${DAS_EXTENDED_CORE} ${DAS_EXTENDED_MODULES} ${DAS_EXTENDED_NIGHTLY}) +# The dasLLVM vector-math rail pins vecmath's answers on Apple silicon; the extended checks have +# no macOS cell, so the darwin build cells run it. +set(DAS_BUILD_LANE_APPLE "") +if(APPLE AND TARGET run_tests_llvm_vector_math AND NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND DAS_BUILD_LANE_APPLE run_tests_llvm_vector_math) +endif() das_lane_group(run_build "Running the build lane" - run_tests_sweeps test-small run_tests_aot_subset) + run_tests_sweeps test-small run_tests_aot_subset ${DAS_BUILD_LANE_APPLE}) das_lane_group(run_build_nightly "Running the build lane's nightly sweeps" run_tests_slow run_backend_sweeps test_aot_subset) das_lane_group(run_build_sanitizers "Running the sanitizer lane" @@ -133,7 +139,7 @@ das_lane_group(run_build_mingw "Running the mingw lane" # The isolated retry of a lane whose sweep failed: the workflow chains ` || _isolated` # (cmake has no retry primitive inside one target). das_lane_group(run_build_isolated "Retrying the build lane's sweeps in isolated mode" - run_tests_sweeps_isolated test-small run_tests_aot_subset) + run_tests_sweeps_isolated test-small run_tests_aot_subset ${DAS_BUILD_LANE_APPLE}) das_lane_group(run_build_sanitizers_isolated "Retrying the sanitizer lane's sweep in isolated mode" run_tests_interpreter_isolated test-small) das_lane_group(run_build_mingw_isolated "Retrying the mingw lane's sweeps in isolated mode" diff --git a/ci/ci_matrix.py b/ci/ci_matrix.py index f90440a3c7..f9e4a5e080 100755 --- a/ci/ci_matrix.py +++ b/ci/ci_matrix.py @@ -15,8 +15,8 @@ LINUX_ARM_EXTRA = CLANG + " -DDAS_GLFW_DISABLED=ON -DDAS_HV_DISABLED=OFF -DDAS_SQLITE_DISABLED=OFF" WINDOWS_EXTRA = ("-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" " -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake") -# dastest's whole-suite watchdogs, raised for the cells whose sweep runs at half speed (windows -# Debug, the sanitizers) and then rides the hosted runner's ~1.4x swing; every other cell keeps +# dastest's whole-suite watchdogs, raised for the cells whose sweep runs at half speed (the +# sanitizers) and then rides the hosted runner's ~1.4x swing; every other cell keeps # tests/CMakeLists.txt's defaults, so a hang there still fails fast SLOW_SWEEP_BUDGETS = " -DDAS_TEST_TIMEOUT=3600 -DDAS_TEST_TIMEOUT_ISOLATED=7200 -DDAS_TEST_TIMEOUT_AOT=3600" @@ -37,7 +37,8 @@ def _cells(target, architecture, presets, **props): def build_cells(nightly): """build.yml: Release + Debug on every platform per PR; the sanitizer cells and windows 64 Debug are nightly-only. A sanitizer cell is three jobs, one per test step (`phase`); - every other cell runs every step (`phase` all).""" + windows 64 Debug compiles and runs no test (`phase` compile); every other cell runs every + step (`phase` all).""" cmake = dict(build_system="cmake", cmake_generator="Ninja") cells = [] cells += _cells("linux", 64, ["Debug", "Release"], release_target="linux", release_arch="x86_64", @@ -62,7 +63,7 @@ def build_cells(nightly): if nightly: cells += _cells("windows", 64, ["Debug"], release_target="windows", release_arch="x86_64", runner="windows-latest", architecture_string="x64", archive_ext="zip", - jit_disabled="ON", nightly_only="ON", cmake_extra=WINDOWS_EXTRA + SLOW_SWEEP_BUDGETS, + jit_disabled="ON", nightly_only="ON", phase="compile", cmake_extra=WINDOWS_EXTRA, **cmake) for san in SANITIZERS: for phase in SANITIZER_PHASES: @@ -79,18 +80,18 @@ def build_cells(nightly): def extended_cells(nightly): - """extended_checks.yml: per PR two darwin15 jobs split by role (core: the tree's own gates and - utils; modules: the module and service suites); the nightly runs every step on linux, darwin15 - and windows in one job each (role all).""" + """extended_checks.yml: per PR two linux jobs split by role (core: the tree's own gates and + utils; modules: the module and service suites); the nightly runs every step on linux and + windows in one job each (role all). No macOS cell: the hosted macOS runner is 3 cores and + 7 GB against linux's 4 and 16, and ran the same steps in about twice the time.""" cmake = dict(build_system="cmake", cmake_generator="Ninja") - darwin = dict(target="darwin15", architecture="arm64", runner="macos-15", architecture_string="arm64", **cmake) + linux = dict(target="linux", architecture=64, runner="ubuntu-latest", **cmake) if nightly: return [ - dict(target="linux", architecture=64, role="all", runner="ubuntu-latest", **cmake), - dict(role="all", **darwin), + dict(role="all", **linux), dict(target="windows", architecture=64, role="all", runner="windows-latest", architecture_string="x64", **cmake), ] - return [dict(role="core", **darwin), dict(role="modules", **darwin)] + return [dict(role="core", **linux), dict(role="modules", **linux)] def main(argv): diff --git a/ci/runner_info.sh b/ci/runner_info.sh new file mode 100644 index 0000000000..3b9135ced3 --- /dev/null +++ b/ci/runner_info.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Prints what the hosted runner gave this job: cores, memory, load, free disk. A job that runs +# several times slower than the night before is read against these lines first. +set -u +os=$(uname -s) +echo "runner: os=$os arch=$(uname -m) name=${RUNNER_NAME:-?} image=${ImageOS:-?} ${ImageVersion:-}" +case "$os" in + Linux) + echo "runner: cpus=$(nproc) model=$(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2- | xargs)" + awk '/MemTotal|MemAvailable|SwapTotal/ { printf "runner: %s %d MiB\n", $1, $2 / 1024 }' /proc/meminfo + echo "runner: load=$(cut -d' ' -f1-3 /proc/loadavg)" + ;; + Darwin) + echo "runner: cpus=$(sysctl -n hw.ncpu) model=$(sysctl -n machdep.cpu.brand_string)" + echo "runner: MemTotal: $(( $(sysctl -n hw.memsize) / 1048576 )) MiB" + page=$(sysctl -n hw.pagesize) + free=$(vm_stat | awk '/Pages free|Pages inactive|Pages speculative/ { gsub("\\.", "", $NF); s += $NF } END { print s }') + echo "runner: MemAvailable: $(( free * page / 1048576 )) MiB" + echo "runner: load=$(sysctl -n vm.loadavg)" + ;; + *) + echo "runner: cpus=${NUMBER_OF_PROCESSORS:-?} model=${PROCESSOR_IDENTIFIER:-?}" + powershell -NoProfile -Command '$o = Get-CimInstance Win32_OperatingSystem; "runner: MemTotal: {0} MiB`nrunner: MemAvailable: {1} MiB" -f [int]($o.TotalVisibleMemorySize / 1024), [int]($o.FreePhysicalMemory / 1024)' || true + ;; +esac +echo "runner: disk $(df -h . | awk 'NR == 2 { print "size=" $2 " free=" $4 }')" diff --git a/ci/test_ci_matrix.py b/ci/test_ci_matrix.py index 0aeda5bce2..2291bfe284 100644 --- a/ci/test_ci_matrix.py +++ b/ci/test_ci_matrix.py @@ -32,7 +32,7 @@ def names(cells): "windows-32-Release-none", "windows-64-Release-none", ]) NIGHTLY_ONLY_BUILD = sorted( - ["windows-64-Debug-none", "linux-64-Release-none-fastmath"] + ["windows-64-Debug-none-compile", "linux-64-Release-none-fastmath"] + ["linux-64-Release-%s-%s" % (san, phase) for san in ("asan", "tsan", "ubsan") for phase in ("tests", "slow", "backend")]) @@ -75,19 +75,38 @@ def test_sanitizer_cells_carry_no_archive_fields(self): def test_only_sanitizer_cells_split_into_phases(self): # build.yml gates its three test steps on the phase; a cell that ran none of them would be - # a build with no signal, and a non-sanitizer cell split would multiply the per-PR matrix + # a build with no signal, and a non-sanitizer cell split would multiply the per-PR matrix. + # windows 64 Debug is the one cell that runs none of the three: its signal is the compile for cell in ci_matrix.build_cells("schedule"): - if cell["sanitizers"] == "none": - self.assertEqual(cell["phase"], "all", cell) - else: + if cell["sanitizers"] != "none": self.assertIn(cell["phase"], ("tests", "slow", "backend"), cell) + elif (cell["target"], cell["cmake_preset"]) == ("windows", "Debug"): + self.assertEqual(cell["phase"], "compile", cell) + else: + self.assertEqual(cell["phase"], "all", cell) + + def test_the_compile_only_cell_has_a_step_that_compiles(self): + # a phase no step's condition names is a job that configures and goes green + text = open(os.path.join(WORKFLOWS, "build_matrix.yml"), encoding="utf-8").read() + block = re.search(r"- name: \"Compile\"\n((?: .*\n)+)", text) + self.assertTrue(block, "build_matrix.yml has no Compile step") + self.assertIn("if: matrix.phase == 'compile'", block.group(1)) + self.assertIn("cmake --build ./build", block.group(1)) + self.assertLess(text.index('- name: "Compile"'), text.index('- name: "Save sccache objects"')) + + def test_every_sccache_slot_is_bounded(self): + # sccache's default bound is 10G a slot; an unbounded slot grows to it and evicts the + # slots the nightly cells save once a day + for name in ("build_matrix.yml", "extended_checks.yml"): + text = open(os.path.join(WORKFLOWS, name), encoding="utf-8").read() + self.assertGreaterEqual(text.count('echo "SCCACHE_CACHE_SIZE='), 2, name) def test_half_speed_sweeps_carry_raised_watchdogs(self): - # the sanitizer cells and windows Debug run their sweeps at half speed and lose the - # default whole-suite watchdog to runner variance; every other cell keeps the defaults, - # so a hang still fails fast where the sweep is quick + # the sanitizer cells run their sweeps at half speed and lose the default whole-suite + # watchdog to runner variance; every other cell keeps the defaults, so a hang still + # fails fast where the sweep is quick for cell in ci_matrix.build_cells(True): - slow = cell["sanitizers"] != "none" or cell["cmake_preset"] == "Debug" + slow = cell["sanitizers"] != "none" for flag in ("-DDAS_TEST_TIMEOUT=3600", "-DDAS_TEST_TIMEOUT_ISOLATED=7200", "-DDAS_TEST_TIMEOUT_AOT=3600"): (self.assertIn if slow else self.assertNotIn)(flag, cell["cmake_extra"], cell) @@ -96,14 +115,21 @@ def test_half_speed_sweeps_carry_raised_watchdogs(self): class ExtendedMatrix(unittest.TestCase): - def test_per_pr_is_two_darwin_roles(self): + def test_per_pr_is_two_linux_roles(self): cells = ci_matrix.extended_cells(False) - self.assertEqual([(c["target"], c["role"]) for c in cells], [("darwin15", "core"), ("darwin15", "modules")]) + self.assertEqual([(c["target"], c["role"]) for c in cells], [("linux", "core"), ("linux", "modules")]) def test_nightly_is_one_full_job_per_platform(self): cells = ci_matrix.extended_cells(True) - self.assertEqual([(c["target"], c["role"]) for c in cells], - [("linux", "all"), ("darwin15", "all"), ("windows", "all")]) + self.assertEqual([(c["target"], c["role"]) for c in cells], [("linux", "all"), ("windows", "all")]) + + def test_the_macos_only_check_rides_the_build_lane(self): + # the extended checks have no macOS cell, so the one check cmake gives to APPLE alone has + # to run where a macOS job still builds per PR + text = open(os.path.join(os.path.dirname(HERE), "ci", "CMakeLists.txt"), encoding="utf-8").read() + block = text.split("set(DAS_BUILD_LANE_APPLE", 1)[1].split("das_lane_group(run_build ", 1) + self.assertIn("run_tests_llvm_vector_math", block[0]) + self.assertIn("${DAS_BUILD_LANE_APPLE}", block[1].split(")", 1)[0]) def test_every_cell_names_a_runner(self): for nightly in (False, True): @@ -153,9 +179,12 @@ def test_every_cache_restore_runs_on_the_cron_too(self): for block in re.findall(r"- name: \"(?:Restore|Cache) [^\"]*\"\n((?: .*\n)+)", text): self.assertNotIn("github.event_name != 'schedule'", block, name) - def test_one_sanitizer_phase_writes_the_shared_slot(self): + def test_each_sanitizer_slot_has_one_writer(self): + # tests and backend share a slot the tests job writes; slow compiles the full test_aot, + # which is in no other job's objects, so it keeps a slot of its own text = self.read("build_matrix.yml") - self.assertIn("github.ref == 'refs/heads/master' && matrix.phase != 'slow' && matrix.phase != 'backend'", text) + self.assertIn("github.ref == 'refs/heads/master' && matrix.phase != 'backend'", text) + self.assertEqual(text.count("${{ matrix.phase == 'slow' && '-slow' || '' }}@"), 4) self.assertNotIn("matrix.nightly_only != 'ON'", text) def test_the_three_test_steps_gate_on_the_phase(self): diff --git a/dastest/README.md b/dastest/README.md index 57c814d97d..f82ce1f7a9 100644 --- a/dastest/README.md +++ b/dastest/README.md @@ -42,7 +42,7 @@ Any other argument, or a wrong type, is a compile error from the macro. - `--test-names `: Run top-level tests matching "namePrefix", such as "namePrefix1" - `--test-list `: Path to a file listing one test path per line; restricts the collected set to those paths - `--test-only `: Restrict the collected set to this test path (repeatable) -- `--exclude `: Skip test files whose base name contains this substring (repeatable) +- `--exclude `: Skip test files whose base name contains this substring; a substring with a `/` is matched against the whole path, so `tests/lint/` skips a folder (repeatable) - `--test-project `: Project file. Will be used to compile given tests - `--module-cache-dir `: Keep AST caches per test file and compile configuration, including isolated workers. `--verbose` prints each verdict. diff --git a/dastest/dastest.das b/dastest/dastest.das index c0b11b0be2..27d78be2ad 100644 --- a/dastest/dastest.das +++ b/dastest/dastest.das @@ -638,7 +638,10 @@ def main() : int { // nolint:STYLE037,STYLE038 - CLI dispatch, one arm per mode if (!empty(test_args.exclude_files)) { files |> erase_if() $(f) { let base = base_name(f) - return (find_index_if(test_args.exclude_files) $(pat) => base |> contains(pat)) >= 0 + let path = f |> replace("\\", "/") + return (find_index_if(test_args.exclude_files) $(pat) { + return (pat |> contains("/") ? path : base) |> contains(pat) + }) >= 0 } } diff --git a/dastest/dastest_clargs.das b/dastest/dastest_clargs.das index edabffa766..aacd50243d 100644 --- a/dastest/dastest_clargs.das +++ b/dastest/dastest_clargs.das @@ -74,7 +74,7 @@ struct DastestArgs { test_files : array @clarg_name = "exclude" - @clarg_doc = "Skip test files whose base name contains this substring (repeatable)" + @clarg_doc = "Skip test files whose base name contains this substring; a substring with a '/' is matched against the whole path, so 'tests/lint/' skips a folder (repeatable)" exclude_files : array @clarg_name = "test-list" diff --git a/site/REVIEW.das b/site/REVIEW.das index 3375921521..76f41bb0b4 100644 --- a/site/REVIEW.das +++ b/site/REVIEW.das @@ -275,6 +275,9 @@ def private check_dl_names_are_defined { // into the playground beside it; the gate compiles each block with the binary running it. // A blog post is prose, not a page here; a block marked data-playground="none" is compiled // but owes no link (the ledgered case: a sample of an engine the wasm build does not carry) +let private SAMPLE_COMPILE_TIMEOUT_S = 300.0 //! under the gate runner's cap +let private SAMPLE_SLOW_S = 10.0lf + def private check_embedded_samples { let argv <- get_command_line_arguments() let self_binary = argv[0] @@ -312,13 +315,19 @@ def private check_embedded_samples { let tmp = unsafe(tmp_r.value) fwrite(tmp, blk.source) var output = "" - let rc = unsafe(popen_argv([self_binary, "-compile-only", tmp], 0.0) $(f) { - if (f != null) { - output = fread(f) - } - }) + let t0 = ref_time_ticks() + var inscope compile <- [self_binary, "-compile-only", tmp] + let rc = run_and_capture(compile, output, SAMPLE_COMPILE_TIMEOUT_S) + let took = double(get_time_usec(t0)) / 1000000.0lf remove(tmp) - if (rc != 0) { + if (took >= SAMPLE_SLOW_S) { + print("site: {page}:{line}: sample {k} took {int(took)} s to compile\n") + } + if (rc == popen_timed_out) { + let cap = int(SAMPLE_COMPILE_TIMEOUT_S) + gate_finding(page, line, + "sample {k} did not compile in {cap} s - a compiler hang, or a slow machine") + } elif (rc != 0) { let first_error = find_first_error_line(output) gate_finding(page, line, "sample {k} does not compile with the current toolchain: {first_error}") } diff --git a/skills/internal/preflight.md b/skills/internal/preflight.md index c5bcc5939d..c5c2ecfb4d 100644 --- a/skills/internal/preflight.md +++ b/skills/internal/preflight.md @@ -61,11 +61,11 @@ working-tree copy. | Workflow | Trigger | Jobs | |---|---|---| | `build.yml` (per-PR) | every PR commit (`pull_request`) + pushes to `master` | `build` matrix (`build_matrix.yml`, lane `build`: Debug + Release on linux, linux_arm, darwin15, darwin26; windows 32 Release, windows 64 Release), `bundle_smoke`, `build_linux_gcc` | -| `nightly.yml` | `schedule` cron (daily 02:00 UTC), `workflow_dispatch`, and a `pull_request` whose body says `#nightly` or that carries `run-nightly` / `nightly-failure` | `build` matrix (`build_matrix.yml`, lane `build_nightly`: the per-PR Release cells running `run_build_nightly` - the full AOT sweep and the three backend sweeps - plus the sanitizer cells - each three jobs, `tests`, `slow`, `backend`, sharing one sccache slot - the fast-math cell and windows 64 Debug), `build_windows_relwithdebinfo_nightly`, `build_windows_release_llvm_nightly`, `build_windows_mingw`, `build_windows_clangcl`, and calls to `extended_checks.yml` (lane `extended_nightly`), `nightly_daspkg_index.yml`, `nightly_imgui.yml`, `nightly_playground.yml` and `nightly_vulkan.yml` - the only cron. Breaks surface within ~24 h, not at PR time | +| `nightly.yml` | `schedule` cron (daily 02:00 UTC), `workflow_dispatch`, and a `pull_request` whose body says `#nightly` or that carries `run-nightly` / `nightly-failure` | `build` matrix (`build_matrix.yml`, lane `build_nightly`: the per-PR Release cells running `run_build_nightly` - the full AOT sweep and the three backend sweeps - plus the sanitizer cells - each three jobs, `tests`, `slow`, `backend`; `tests` and `backend` share one sccache slot, `slow` keeps its own - the fast-math cell and windows 64 Debug, which compiles and runs no test), `build_windows_relwithdebinfo_nightly`, `build_windows_release_llvm_nightly`, `build_windows_mingw`, `build_windows_clangcl`, and calls to `extended_checks.yml` (lane `extended_nightly`), `nightly_daspkg_index.yml`, `nightly_imgui.yml`, `nightly_playground.yml` and `nightly_vulkan.yml` - the only cron. Breaks surface within ~24 h, not at PR time | | `build_matrix.yml` | `workflow_call` from `build.yml` and `nightly.yml`, with the lane | the build matrix job - section below | | `nightly_imgui.yml` | `workflow_call` from `nightly.yml` + `workflow_dispatch` | dasImgui playwright suite on ubuntu + macos - section below | -| `extended_checks.yml` (per-PR) | every PR | two darwin15-arm64 jobs, `core` and `modules` (`ci/ci_matrix.py extended`), ALL release modules ON, plus `commit_messages` (ubuntu, no build) - section below | -| `extended_checks.yml` (nightly) | `workflow_call` from `nightly.yml` | one job each on linux, darwin15 and windows running every step (role `all`), including the ones too slow for a PR: tutorial dry-runs, the run form of examples, coverage, the nano cross-compile, the AST verify tree sweep, doc-verify | +| `extended_checks.yml` (per-PR) | every PR | two linux x86_64 jobs, `core` and `modules` (`ci/ci_matrix.py extended`), ALL release modules ON, plus `commit_messages` (ubuntu, no build) - section below | +| `extended_checks.yml` (nightly) | `workflow_call` from `nightly.yml` | one job each on linux and windows running every step (role `all`), including the ones too slow for a PR: tutorial dry-runs, the run form of examples, coverage, the nano cross-compile, the AST verify tree sweep, doc-verify | | `wasm_build.yml` | every PR | emscripten build of `web/` on 3 OSes, the vecmath backend battery and `tests/language` under node, + `wasm_cross` | | `build_eastl.yml` | every PR | EASTL shadow-config build + no-fileio build (linux clang) | | `doc.yml` | only if `doc/**`, `daslib/**`, `src/builtin/**`, `modules/dasImgui/**`, `modules/dasVulkan/**`, or `modules/dasLLAMA/dasllama/**` changed; `release: prereleased` of a daslang tag (a `dasllama-*` tag runs nothing) | the doc gates | @@ -90,10 +90,14 @@ part of ALL) as a compile+link gate, and run no AOT tests. **The build matrix runs `run_build` (`run_tests_sweeps`, `test-small`), retried as `run_build_isolated` (`run_tests_sweeps_isolated`, `test-small`) when it fails, and, on the nightly lane, `run_build_nightly` (`run_tests_slow`, `run_backend_sweeps`, `test_aot_subset`).** The retry -is the workflow's `||`, not a cmake target. The windows 64 Debug cell and the sanitizer cells +is the workflow's `||`, not a cmake target. The sanitizer cells configure with raised watchdogs (`SLOW_SWEEP_BUDGETS` in `ci/ci_matrix.py`: 3600 s for a sweep and the full AOT suite, 7200 s isolated) - their sweeps run at half speed on their own and the -default budgets fired on runner variance alone. Which sweeps +default budgets fired on runner variance alone. The windows 64 Debug cell runs the `Compile` +step alone (`phase` compile in `ci/ci_matrix.py`): the default build, no sweep. Every cell's +sccache slot is bounded by `SCCACHE_CACHE_SIZE` (5000M on windows, 2000M elsewhere) - an +unbounded slot grows to sccache's 10G default and evicts the slots the nightly cells save once +a day. Which sweeps `run_tests_sweeps` runs is decided in cmake, not in the workflow: no JIT where dasLLVM is off (the sweep targets live under `if(NOT DAS_LLVM_DISABLED)`) or on Debug; no interpreter sweep under a sanitizer. @@ -110,7 +114,8 @@ configure. | Interpreter sweep | `cmake --build build --config Release --target run_tests_sweeps` - the whole per-PR sweep, JIT then interpreter, each retried isolated; alone: `cmake --build build --config Release --target run_tests_interpreter` | fails if a completed test file exceeded 30 s | | JIT sweep | `cmake --build build --config Release --target run_tests_jit` (`run_tests_sweeps` runs it first, then the interpreter sweep) | Windows-local `clang-cl` link failures are env noise; the catchable class is LLVM verifier errors, end-to-end JIT needs WSL/mac. 2-test smoke: `skills/internal/make_pr.md` sec.2.5 | | Small C++ tests | `cmake --build build --config Release --target test-small` | drop `--build-config` on single-config generators. **Run after touching `tests-cpp/`** - MSVC tolerates C++ clang/gcc reject; `skills/internal/writing_cpp_tests.md` | -| AOT sweep (full) | `cmake --build build --config Release --target run_tests_slow` - the full `test_aot` build, the AOT sweep over tests/, then the big C++ tests | nightly + manual dispatch only, so this is the **only** pre-push gate for AOT regressions outside tests/language - don't skip it | +| Compile (windows 64 Debug, nightly) | `cmake --build build --config Debug` | the default build as MSVC Debug; no test runs in this cell | +| AOT sweep (full) | `cmake --build build --config Release --target run_tests_slow` - the full `test_aot` build, the AOT sweep over tests/, then the big C++ tests | nightly + manual dispatch only, so this is the **only** pre-push gate for AOT regressions outside tests/language - don't skip it; the sweep closes with its 30 slowest files (`--timing-outliers 30`). Under a sanitizer it leaves out `tests/lsp`, `tests/lint`, `tests/module_cache`, `tests/dastest` and `tests/mcp` - compiler and tool tests the cell's interpreter sweep already runs | | AOT subset gate | `cmake --build build --config Release --target test_aot_subset` (add `--target run_tests_aot_subset` to sweep tests/language too) | what per-PR lanes build | | LLVM-AOT sweep | `cmake --build build --config Release --target run_backend_sweeps` runs all three; alone: `cmake --build build --config Release --target run_tests_llvm_aot` | nightly + dispatch. Emits a native `.o` per test through the LLVM backend instead of the C++ proxy, then sweeps tests/ against them - tests/language only under tsan, where the whole corpus outruns the suite watchdog | | Standalone sweep - C++ (`-ctx`) | `cmake --build build --config Release --target run_backend_sweeps`; alone: `cmake --build build --config Release --target run_standalone_sweep_aot` | nightly + dispatch. Emits, compiles, links and launches a standalone context per file of test_aot's corpus. `-DDAS_STANDALONE_SWEEP_FILTER=` narrows it | @@ -167,20 +172,22 @@ headers live there, and TUs like `src/simulate/fs_file_info.cpp` include them. ## extended_checks.yml -Per PR the workflow is two darwin15 jobs, `extended_checks (darwin15, core)` and -`extended_checks (darwin15, modules)`, each inside the 35-minute budget: `core` runs the +Per PR the workflow is two linux jobs, `extended_checks (linux, core)` and +`extended_checks (linux, modules)`, each inside the 35-minute budget; no cell runs on macOS, +whose hosted runner (3 cores, 7 GB) took about twice linux's time on the same steps: `core` runs the tree's own gates - dasgen, utils tests, standalone exes, formatter, lint, line length, ast-verify, the python gates, review-md discovery, dastest's own suite, the REVIEW.das gates, ci-das - and `modules` the module and service suites - ser/deser, MCP tools, boulder-dash, -dasllama-server, env-knob registries, the dasLLVM vector-math rail, facade lint, +dasllama-server, env-knob registries, facade lint, dasweb-playground, dasllama-ladder, dasweb-buildd, dasweb-verify, the sequence smoke, the dasGlfw touch layer, the daslang_static sweep. The role names a group target, so the nightly job -(role `all`, one each on linux, darwin15, windows) runs every check plus the +(role `all`, one each on linux and windows) runs every check plus the nightly-only ones: tutorial dry-runs and the run form of examples (preflight's `compile-sweep` is their per-PR mirror), the AST verify tree sweep and doc-verify (policy), coverage (a report, -not a gate), and the nano cross-compile - platform-bound: its arm-none-eabi toolchain is an apt -package no darwin cell and no developer box carries, so it has no per-PR cell and no local -mirror. The cells are `ci/ci_matrix.py extended `; `ci/test_ci_matrix.py` pins them and +not a gate), and the nano cross-compile - its arm-none-eabi toolchain is an apt +package no developer box carries, so it has no local mirror. The dasLLVM vector-math rail is +Apple-only and runs in the darwin build cells' `run_build` lane (`DAS_BUILD_LANE_APPLE` in +`ci/CMakeLists.txt`). The cells are `ci/ci_matrix.py extended `; `ci/test_ci_matrix.py` pins them and the condition spelling. **CI configures with ALL release modules ON** - `ci/release_modules.txt` flips @@ -239,11 +246,11 @@ test rather than a check that silently stopped running. | dasllama-convert carrier sniff | `cmake --build build --config Release --target run_tests_dasllama_convert` | the `modules` role; a cell whose model is absent skips | | dasOPENAI client tests | `cmake --build build --config Release --target run_tests_dasopenai` | the `modules` role; a cell that talks to a server the repo does not start skips without it | | env-knob registries | `cmake --build build --config Release --target run_tests_dasllama_env_registry` | | -| dasLLVM inline-polynomial rail | `cmake --build build --config Release --target run_tests_llvm_vector_math` | the darwin15 cell pins vecmath's answers; needs `-jit` | +| dasLLVM inline-polynomial rail | `cmake --build build --config Release --target run_tests_llvm_vector_math` | the darwin build cells run it inside `run_build` and pin vecmath's answers; needs `-jit` | | dasllama facade lint | `cmake --build build --config Release --target run_tests_dasllama_facade_lint` | the DASLLAMA001 smokes | | dasweb-playground / ladder / buildd / verify | `cmake --build build --config Release --target run_tests_dasweb_playground`, `run_tests_dasllama_ladder`, `run_tests_dasweb_buildd`, `run_tests_dasweb_verify` | in-dir suites `--test ./tests` never reaches; each server suite binds a loopback port and shuts itself down | | CI-only das surface | `cmake --build build --config Release --target check_gated_das` | reads the same `ci_only_das.txt` preflight's `ci-das` gate does | -| nano cortex-m4 cross-compile - **nightly** (linux) | `cmake --build build --config Release --target check_nano_arm` | its arm-none-eabi toolchain is an apt package no darwin cell and no developer box carries | +| nano cortex-m4 cross-compile - **nightly** (linux) | `cmake --build build --config Release --target check_nano_arm` | its arm-none-eabi toolchain is an apt package no developer box carries | | Bundle smoke (build.yml) | `cmake --build build --config Release --target check_bundle_smoke` | installs to `/daslang_bundle` and runs `ci/smoke_test_bundle.sh` | | dasClangBind self-binder (mingw nightly) | `DAS_CLANG_INCLUDE= cmake --build build --config Release --target check_clangbind_fresh`, then `run_tests_cbind_const_preproc` | needs `DAS_CLANG_BIND_DISABLED=OFF` | | dasGlfw touch layer | `cmake --build build --config Release --target run_tests_glfw_touch` | the `modules` role; the EM_JS bodies run under node against fake GLFW/Browser fixtures, no browser and no window | diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 278bf098f2..cbc5c33c89 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -121,8 +121,17 @@ endif() # The full binary is EXCLUDE_FROM_ALL; this target builds it on demand (nightly CI, # preflight --full). if(TARGET test_aot) + # Under a sanitizer the sweep leaves out the folders that test the compiler and the tools: + # their files compile programs and launch daslang children, so their time is the compiler's, + # multiplied by the sanitizer, and the interpreter sweep of the same cell already runs them. + set(_DAS_AOT_SWEEP_EXCLUDE "") + if(DAS_USE_SANITIZER AND NOT DAS_USE_SANITIZER STREQUAL "none") + foreach(_dir lsp lint module_cache dastest mcp) + list(APPEND _DAS_AOT_SWEEP_EXCLUDE --exclude tests/${_dir}/) + endforeach() + endif() add_das_target(run_tests_aot - COMMAND ${DAS_SANITIZER_ENV} $ -use-aot ${PROJECT_SOURCE_DIR}/dastest/dastest.das -- --use-aot --color --failures-only --timeout ${DAS_TEST_TIMEOUT_AOT} --test ${PROJECT_SOURCE_DIR}/tests + COMMAND ${DAS_SANITIZER_ENV} $ -use-aot ${PROJECT_SOURCE_DIR}/dastest/dastest.das -- --use-aot --color --failures-only --timing-outliers 30 --timeout ${DAS_TEST_TIMEOUT_AOT} ${_DAS_AOT_SWEEP_EXCLUDE} --test ${PROJECT_SOURCE_DIR}/tests DEPENDS test_aot WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMENT "Running tests (AOT)" diff --git a/tests/dastest/test_json_output.das b/tests/dastest/test_json_output.das index 636c29bdeb..b5379e9083 100644 --- a/tests/dastest/test_json_output.das +++ b/tests/dastest/test_json_output.das @@ -152,6 +152,14 @@ def test_test_only_filter(tt : T?) { tt |> equal(run_dastest("pass", "--test-only {fixture}.other").total, 0) } +[test(description = "--exclude matches a plain substring against the base name alone and a substring with a slash against the whole path")] +def test_exclude_filter(tt : T?) { + tt |> equal(run_dastest("pass", "--exclude fixture_pass").total, 0) + tt |> equal(run_dastest("pass", "--exclude dastest").total, 2, "a bare folder name") + tt |> equal(run_dastest("pass", "--exclude tests/dastest/").total, 0) + tt |> equal(run_dastest("pass", "--exclude tests/nosuchfolder/").total, 2) +} + [test(description = "--test-names keeps only the tests whose names it lists")] def test_test_name_filter(tt : T?) { let report = run_dastest("pass", "--test-names test_passing") diff --git a/utils/internal/review-md/gate.das b/utils/internal/review-md/gate.das index 17099247bb..b4ba3ad72b 100644 --- a/utils/internal/review-md/gate.das +++ b/utils/internal/review-md/gate.das @@ -27,6 +27,7 @@ def walker_self_binary : string { } let GATE_TIMEOUT_S = 600.0 //! the site gate compiles a dasLLAMA sample from source per page: over a minute on the slowest CI runner, and a killed gate read as red with no output +let GATE_SLOW_S = 10.0lf //! Execute each gate (repo-relative path), printing its output. Returns the red-gate count. def run_gates(root : string; gates : array) : int { @@ -40,8 +41,13 @@ def run_gates(root : string; gates : array) : int { var failed = 0 for (g in gates) { var out : string + let t0 = ref_time_ticks() let rc = run_and_capture([self, g], out, GATE_TIMEOUT_S) + let took = double(get_time_usec(t0)) / 1000000.0lf print(out) + if (took >= GATE_SLOW_S) { + print("REVIEW.das {g}: {int(took)} s\n") + } if (rc != 0) { failed++ if (empty(out)) {