[CI] Fix the CI jobs that fail on the OSDC images

.github/unittest/examples/scripts/test_examples.py

@@ -650,6 +650,14 @@ def test_run_example_terminates_timeouts(tmp_path: Path) -> None:
             os.kill(child_pid, 0)
         except ProcessLookupError:
             break
+        # A killed child re-parented to a PID 1 that never reaps orphans (OSDC's
+        # `tail -f /dev/null`) stays a zombie, which os.kill still reports.
+        try:
+            stat = Path(f"/proc/{child_pid}/stat").read_text()
+            if stat.rsplit(")", 1)[1].split()[0] == "Z":
+                break
+        except (FileNotFoundError, IndexError):
+            pass
         time.sleep(0.01)
     else:
         pytest.fail(f"child process {child_pid} survived timeout cleanup")

.github/unittest/helpers/reap_strays.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# On OSDC the step runs under run_with_env_secrets.py, which drains our stdout
+# until EOF. EOF only arrives once every process holding the write end has
+# closed it, so a single xdist worker, Ray actor or Xvfb that outlives the tests
+# keeps the job alive until its 120-minute timeout, long after the script exits.
+# Under linux_job_v2 the surrounding `docker run` reaped these for us.
+
+echo "::group::Processes still alive before exit"
+ps -eo pid,ppid,etimes,rss,args --sort=-rss | head -40 || true
+echo "::endgroup::"
+
+# Anything still running that is not this shell or ps itself.
+strays="$(pgrep -f 'pytest|ray::|Xvfb' 2>/dev/null | grep -v "^$$\$" || true)"
+if [ -n "${strays}" ]; then
+  echo "Reaping strays holding the step open: ${strays}"
+  # shellcheck disable=SC2086
+  kill -TERM ${strays} 2>/dev/null || true
+  sleep 5
+  # shellcheck disable=SC2086
+  kill -KILL ${strays} 2>/dev/null || true
+fi

.github/unittest/linux/scripts/run_all.sh

@@ -618,25 +618,7 @@ bash ${this_dir}/post_process.sh
 # ==================================================================================== #
 # ================================ Reap strays ======================================= #
 
-# On OSDC the step runs under run_with_env_secrets.py, which drains our stdout
-# until EOF. EOF only arrives once every process holding the write end has
-# closed it, so a single xdist worker or Ray actor that outlives pytest keeps
-# the job alive until its 120-minute timeout, long after this script exits 0.
-# Under linux_job_v2 the surrounding `docker run` reaped these for us.
-echo "::group::Processes still alive before exit"
-ps -eo pid,ppid,etimes,rss,args --sort=-rss | head -40 || true
-echo "::endgroup::"
-
-# Anything still running that is not this shell or ps itself.
-strays="$(pgrep -f 'pytest|ray::|Xvfb' 2>/dev/null | grep -v "^$$\$" || true)"
-if [ -n "${strays}" ]; then
-  echo "Reaping strays holding the step open: ${strays}"
-  # shellcheck disable=SC2086
-  kill -TERM ${strays} 2>/dev/null || true
-  sleep 5
-  # shellcheck disable=SC2086
-  kill -KILL ${strays} 2>/dev/null || true
-fi
+bash "${root_dir}/.github/unittest/helpers/reap_strays.sh"
 
 # Exit with failure if any tests failed
 exit $EXIT_STATUS

.github/unittest/linux_libs/scripts_craftground/run_test.sh

@@ -26,6 +26,7 @@ java -version
 # The Minecraft client needs an X display; use a virtual framebuffer with
 # Mesa software rendering.
 Xvfb :99 -screen 0 1024x768x24 &
+trap 'bash .github/unittest/helpers/reap_strays.sh' EXIT
 export DISPLAY=:99
 
 python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/libs/test_craftground.py --instafail -v --durations 200 --capture no -k TestCraftGround --error-for-skips --runslow

.github/unittest/linux_libs/scripts_habitat/run_all.sh

@@ -12,7 +12,7 @@ apt-get install -y vim git wget cmake ninja-build
 # and both resolve on jammy too.
 apt-get install -y libglfw3 libglfw3-dev libgl1 libglx-mesa0 libosmesa6 libosmesa6-dev libglew-dev
 apt-get install -y libxinerama-dev libxcursor-dev libxi-dev libxrandr-dev libxxf86vm-dev
-apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2
+apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 libglib2.0-0
 apt-get install -y libegl1-mesa-dev libgles2-mesa-dev
 
 # Build tools and libraries for habitat-sim

.github/unittest/linux_libs/scripts_habitat/setup_env.sh

@@ -89,11 +89,12 @@ cd habitat-sim
 
 # Build with headless (EGL) and bullet physics support
 # Ensure system cmake is used (pip cmake 4.x is incompatible with habitat-sim's CMake files)
-# Put /usr/bin at the front of PATH to prefer system cmake over any pip-installed cmake
-export PATH="/usr/bin:$PATH"
+# Put /usr/bin at the front of PATH for this build only to prefer system cmake
+# over any pip-installed cmake; exporting it would make later pip3 calls resolve
+# to the externally-managed system pip instead of the conda env's
 # Also set CMAKE_EXECUTABLE to explicitly use system cmake
 export CMAKE_EXECUTABLE=/usr/bin/cmake
-pip3 install . --no-build-isolation
+PATH="/usr/bin:$PATH" "${env_dir}/bin/pip3" install . --no-build-isolation
 
 cd "${root_dir}"
 

.github/unittest/linux_libs/scripts_libero/run_test.sh

@@ -22,6 +22,7 @@ timeout 300s python -m torch.utils.collect_env
 git config --global --add safe.directory '*'
 
 Xvfb :99 -screen 0 1024x768x24 &
+trap 'bash .github/unittest/helpers/reap_strays.sh' EXIT
 
 timeout 120s python -c 'import torch; t = torch.ones([2, 2], device="cuda:0" if torch.cuda.is_available() else "cpu"); print(t); print("tensor device:" + str(t.device))'
 timeout 120s python -c 'from torchrl.envs.libs.libero import _ensure_libero_config; _ensure_libero_config()'

.github/unittest/linux_libs/scripts_minari/setup_env.sh

@@ -8,7 +8,8 @@
 set -e
 set -v
 
-apt-get update && apt-get upgrade -y && apt-get install -y git cmake
+# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI
+apt-get update && apt-get upgrade -y && apt-get install -y git cmake python3-venv
 # Avoid error: "fatal: unsafe repository"
 git config --global --add safe.directory '*'
 apt-get install -y wget \
@@ -26,7 +27,8 @@ apt-get install -y wget \
     libgl1 \
     libglx0 \
     libegl1 \
-    libgles2
+    libgles2 \
+    python3-dev
 
 # Upgrade specific package
 apt-get upgrade -y libstdc++6

.github/unittest/linux_libs/scripts_openx/setup_env.sh

@@ -8,7 +8,8 @@
 set -e
 set -v
 
-apt-get update && apt-get upgrade -y && apt-get install -y git cmake
+# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI
+apt-get update && apt-get upgrade -y && apt-get install -y git cmake python3-venv
 # Avoid error: "fatal: unsafe repository"
 git config --global --add safe.directory '*'
 apt-get install -y wget \

.github/unittest/linux_libs/scripts_procgen/setup_env.sh

@@ -27,7 +27,8 @@ apt-get install -y wget \
     libglx0 \
     libegl1 \
     libgles2 \
-    libglib2.0-0
+    libglib2.0-0 \
+    python3-dev
 
 # Upgrade specific package
 apt-get upgrade -y libstdc++6

.github/unittest/linux_libs/scripts_safety_gymnasium/setup_env.sh

@@ -27,7 +27,8 @@ apt-get install -y wget \
     libglx0 \
     libegl1 \
     libgles2 \
-    libglib2.0-0
+    libglib2.0-0 \
+    python3-dev
 
 # Upgrade specific package
 apt-get upgrade -y libstdc++6

.github/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh

@@ -12,6 +12,11 @@ python -m torch.utils.collect_env
 # Avoid error: "fatal: unsafe repository"
 git config --global --add safe.directory '*'
 
+# Resolve the OSDC container's hostname for torchrl/_comm/distributed.py; see
+# linux/scripts/run_all.sh.
+echo "127.0.0.1 $(hostname)" >> /etc/hosts
+getent hosts "$(hostname)"
+
 root_dir="$(git rev-parse --show-toplevel)"
 env_dir="${root_dir}/env"
 lib_dir="${env_dir}/lib"

.github/unittest/llm/scripts_llm/setup_env.sh

@@ -9,13 +9,10 @@ set -e
 export DEBIAN_FRONTEND=noninteractive
 export TZ=UTC
 apt-get update
-apt-get install -yq --no-install-recommends git wget unzip curl patchelf
+# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI
+apt-get install -yq --no-install-recommends git wget unzip curl patchelf cmake python3-venv
 # Avoid error: "fatal: unsafe repository"
 git config --global --add safe.directory '*'
-# The base PyTorch devel image provides compilers, CMake >= 3.22, and most build deps.
-# Install only minimal utilities not guaranteed to be present.
-
-# CMake available in the PyTorch devel image (Ubuntu 22.04) is sufficient.
 
 # Cleanup APT cache
 apt-get clean && rm -rf /var/lib/apt/lists/*

.github/unittest/llm/scripts_sglang/install.sh

@@ -69,7 +69,7 @@ fi
 printf "* Installing tensordict\n"
 # Install tensordict dependencies first (pyvers is required but --no-deps skips it)
 uv pip install cloudpickle packaging importlib_metadata numpy orjson "pyvers>=0.2.3,<0.3.0"
-uv pip install "pybind11[global]" ninja
+uv pip install "pybind11[global]" ninja cmake
 if [[ "$RELEASE" == 0 ]]; then
     uv pip install --no-build-isolation --no-deps git+https://github.com/pytorch/tensordict.git@$(cat .github/ci_commit_pins/tensordict.txt)
 else

.github/unittest/llm/scripts_sglang/setup_env.sh

@@ -8,7 +8,8 @@ export DEBIAN_FRONTEND=noninteractive
 export TZ=UTC
 
 apt-get update
-apt-get install -yq --no-install-recommends git wget unzip curl patchelf
+# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI
+apt-get install -yq --no-install-recommends git wget unzip curl patchelf python3-dev python3-venv
 # Avoid error: "fatal: unsafe repository"
 git config --global --add safe.directory '*'