From 87a5feb5ff0c950ede4dec766081610699dc214b Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 29 Jul 2026 09:37:54 +1200 Subject: [PATCH 1/3] [ML] Run QA and PyTorch suites concurrently in a single triggered build When both ci:run-qa-tests and ci:run-pytorch-tests are requested, emit a single Appex QA trigger passing a comma-separated QAF_TESTS_TO_RUN so the appex-qa generator fans them into concurrent parallel jobs, instead of two separate triggered builds that collided under the appex pipeline's skip_queued_branch_builds (one was skipped). Single-suite behaviour is unchanged. --- .buildkite/pipeline.json.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 13b22c2df..a12dbdb9d 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -80,13 +80,22 @@ def main(): ".buildkite/pipelines/run_es_tests_x86_64.yml.sh")) pipeline_steps.append(pipeline_steps.generate_step("Upload ES inference tests x86_64 runner pipeline", ".buildkite/pipelines/run_es_inference_tests_x86_64.yml.sh")) - # We only use linux x86_64 builds for QA tests. - if config.run_qa_tests: + # We only use linux x86_64 builds for QA tests. When both the QA + # (ml_cpp_pr) and PyTorch suites are requested we trigger a SINGLE + # downstream build with a comma-separated QAF_TESTS_TO_RUN so the + # appex-qa generator fans them out into concurrent parallel jobs. + # A single trigger avoids the same-branch build de-duplication + # (skip_queued_branch_builds) on the appex pipeline that otherwise + # skipped one of two separately-triggered builds. + if config.run_qa_tests or config.run_pytorch_tests: + qa_suites = [] + if config.run_qa_tests: + qa_suites.append(os.environ.get("QAF_TESTS_TO_RUN") or "ml_cpp_pr") + if config.run_pytorch_tests: + qa_suites.append("pytorch_tests") + env["QAF_TESTS_TO_RUN"] = ",".join(qa_suites) pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline", ".buildkite/pipelines/run_qa_tests.yml.sh")) - if config.run_pytorch_tests: - pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline", - ".buildkite/pipelines/run_pytorch_tests.yml.sh")) if config.build_aarch64 and not config.skip_version_bump_pr_ci: pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", ".buildkite/pipelines/run_es_tests_aarch64.yml.sh")) From 5f74468dcf5c2008bd2387293af8735132170a26 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 29 Jul 2026 11:20:51 +1200 Subject: [PATCH 2/3] Address Copilot review: de-dup QAF_TESTS_TO_RUN suites and rename trigger step Co-authored-by: Cursor --- .buildkite/pipeline.json.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index a12dbdb9d..785485fb5 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -90,11 +90,20 @@ def main(): if config.run_qa_tests or config.run_pytorch_tests: qa_suites = [] if config.run_qa_tests: - qa_suites.append(os.environ.get("QAF_TESTS_TO_RUN") or "ml_cpp_pr") + # QAF_TESTS_TO_RUN may itself be a comma-separated override + # (e.g. a subset of QA markers), so split it into individual + # suites rather than appending the whole string verbatim. + override = os.environ.get("QAF_TESTS_TO_RUN") + if override: + qa_suites.extend(s.strip() for s in override.split(",") if s.strip()) + else: + qa_suites.append("ml_cpp_pr") if config.run_pytorch_tests: qa_suites.append("pytorch_tests") - env["QAF_TESTS_TO_RUN"] = ",".join(qa_suites) - pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline", + # De-duplicate while preserving order so an override that already + # includes pytorch_tests does not produce a duplicate suite. + env["QAF_TESTS_TO_RUN"] = ",".join(dict.fromkeys(qa_suites)) + pipeline_steps.append(pipeline_steps.generate_step("Upload QA/PyTorch tests runner pipeline", ".buildkite/pipelines/run_qa_tests.yml.sh")) if config.build_aarch64 and not config.skip_version_bump_pr_ci: pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", From cf85e10f0ca42afb3ee1bf33ffdc3c652df92c4b Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 31 Jul 2026 13:18:18 +1200 Subject: [PATCH 3/3] [ML] Derive QA/PyTorch trigger label and status context from suites The consolidated single trigger uploaded run_qa_tests.yml.sh for every combination, so a PyTorch-only or combined run was mislabelled "QA Tests" and posted the QA commit-status context. Derive a descriptor (QA / PyTorch / QA + PyTorch) from QAF_TESTS_TO_RUN and use it for the step label, echo and github_commit_status context. Substring matching keeps it correct for marker expressions like "ml_cpp_pr and not slow". Addresses a Copilot review note on #3131. Co-authored-by: Cursor --- .buildkite/pipelines/run_qa_tests.yml.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipelines/run_qa_tests.yml.sh b/.buildkite/pipelines/run_qa_tests.yml.sh index 3c6ddaaa0..ead10bced 100755 --- a/.buildkite/pipelines/run_qa_tests.yml.sh +++ b/.buildkite/pipelines/run_qa_tests.yml.sh @@ -18,16 +18,34 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # shellcheck source=/dev/null source "${SCRIPT_DIR}/derive_qa_stack_env.sh" +# Derive a human-readable descriptor for the trigger step's label and GitHub +# commit-status context from the requested suites. This single script now +# handles QA-only, PyTorch-only and combined runs, so a fixed "QA Tests" label +# would mislabel the other two. Substring matches (rather than exact tokens) +# keep this correct for marker expressions such as "ml_cpp_pr and not slow". +QAF_SUITES="${QAF_TESTS_TO_RUN:-ml_cpp_pr}" +_has_qa=false +_has_pytorch=false +case "${QAF_SUITES}" in *ml_cpp_pr*) _has_qa=true ;; esac +case "${QAF_SUITES}" in *pytorch_tests*) _has_pytorch=true ;; esac +if [ "${_has_qa}" = true ] && [ "${_has_pytorch}" = true ]; then + QA_TESTS_DESC="QA + PyTorch" +elif [ "${_has_pytorch}" = true ]; then + QA_TESTS_DESC="PyTorch" +else + QA_TESTS_DESC="QA" +fi + cat <