ci: Run x86_64-linux-android tests on Cuttlefish virtual devices

.github/workflows/ci.yaml

@@ -236,6 +236,9 @@ jobs:
           - target: x86_64-apple-darwin
             os: macos-26-intel
           - target: x86_64-linux-android
+            env: { TEST_CUTTLEFISH: 1 }
+            # Keep in sync with the Android build pinned in ci/cuttlefish-setup.sh
+            artifact-tag: android17
           # FIXME: Exec format error (os error 8)
           # - target: x86_64-unknown-linux-gnux32
           - target: x86_64-unknown-linux-musl

ci/README.md

@@ -31,8 +31,12 @@ builds are run on stable/beta/nightly, but are the only ones that do so.
 
 The remaining architectures look like:
 
-* Android runs in a [docker image][android-docker] with an emulator, the NDK,
-  and the SDK already set up. The entire build happens within the docker image.
+* Android x86_64 tests run on a [Cuttlefish](https://source.android.com/docs/devices/cuttlefish)
+  virtual device booted on the CI host by `cuttlefish-setup.sh`; the
+  [docker image][android-docker] cross-compiles the tests with the NDK and
+  drives the device over adb. The remaining Android targets run in docker
+  images with the legacy SDK emulator, the NDK, and the SDK already set up,
+  with the entire build happening within the docker image.
 * The MIPS, ARM, and AArch64 builds all use the QEMU userspace emulator to run
   the generated binary to actually verify the tests pass.
 * The MUSL build just has to download a MUSL compiler and target libraries and

ci/create-artifacts.py

@@ -32,11 +32,20 @@ def main():
     archive_name = f"archive-{now}"
     archive_path = f"{archive_name}.tar.gz"
 
-    sp.run(
-        ["tar", "czvf", archive_path, "-C", build_dir, "-T-"],
-        input=file_list,
-        check=True,
-    )
+    # --dereference because kernel.log is a symlink into logs/
+    tar_cmd = ["tar", "czvf", archive_path, "--dereference", "-C", build_dir, "-T-"]
+
+    # Grab the runtime logs of the Cuttlefish device launched by ci/cuttlefish-setup.sh.
+    # The device runs inside the test container, which keeps it under the bind-mounted
+    # target/ so the logs are visible here.
+    cf_dir = Path(os.getenv("CUTTLEFISH_DIR", "target/cuttlefish")).resolve()
+    cf_runtime = cf_dir / "cuttlefish" / "instances" / "cvd-1"
+    cf_logs = ["logs/logcat", "logs/launcher.log", "kernel.log"]
+    cf_found = [log for log in cf_logs if cf_runtime.joinpath(log).exists()]
+    if cf_found:
+        tar_cmd += ["-C", str(cf_runtime), *cf_found]
+
+    sp.run(tar_cmd, input=file_list, check=True)
 
     # If we are in GHA, set these env vars for future use
     gh_env = os.getenv("GITHUB_ENV")

ci/cuttlefish-setup.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Fetch and boot a Cuttlefish Android virtual device, heavily inspired by
+# Mesa's CI:
+# https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/.gitlab-ci/cuttlefish-runner.sh
+#
+# Runs unprivileged (cvd refuses to run as root) inside the test container,
+# invoked by cuttlefish-entrypoint.sh after it has prepared the devices,
+# networking and groups (see ci/docker/x86_64-linux-android/).
+build="${CUTTLEFISH_BUILD:-15660610}"
+target="${CUTTLEFISH_TARGET:-aosp_cf_x86_64_only_phone-userdebug}"
+cf_dir="${CUTTLEFISH_DIR:-$HOME/cuttlefish}"
+
+mkdir -p "$cf_dir"
+cvd fetch --default_build="${build}/${target}" --target_directory="$cf_dir"
+
+cd "$cf_dir"
+
+# -daemon exits only once the guest has fully booted.
+# -enable_sandbox=false is needed because crosvm's minijail device sandbox
+#  (auto-enabled when /var/empty exists) requires unprivileged user
+#  namespaces, which Ubuntu 24.04+ blocks via AppArmor by default.
+HOME="$cf_dir" timeout 10m ./bin/launch_cvd \
+    -daemon \
+    -enable_sandbox=false \
+    -verbosity=INFO \
+    -file_verbosity=DEBUG \
+    -enable_audio=false \
+    -enable_bootanimation=false \
+    -enable_minimal_mode=true \
+    -enable_modem_simulator=false \
+    -enable_wifi=false \
+    -report_anonymous_usage_stats=no \
+    -cpus=2 \
+    -memory_mb=4096
+
+# Check the guest's adbd is reachable on the port the test runner uses,
+# and log the Android version actually being tested.
+HOME="$cf_dir" ./bin/adb connect 127.0.0.1:6520
+HOME="$cf_dir" ./bin/adb -s 127.0.0.1:6520 wait-for-device
+HOME="$cf_dir" ./bin/adb devices
+HOME="$cf_dir" ./bin/adb -s 127.0.0.1:6520 shell \
+    getprop ro.build.version.release

ci/docker/x86_64-linux-android/Dockerfile

@@ -1,26 +1,38 @@
 FROM ubuntu:26.04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    adb \
     ca-certificates \
+    curl \
     gcc \
     libc-dev \
     python3 \
     unzip \
     wget
 
+# Cuttlefish host packages: the virtual device the tests run on boots
+# inside this container (see cuttlefish-entrypoint.sh for the docker run
+# flags this needs).
+RUN curl -fsSL https://us-apt.pkg.dev/doc/repo-signing-key.gpg \
+        -o /etc/apt/trusted.gpg.d/artifact-registry.asc \
+    && echo "deb https://us-apt.pkg.dev/projects/android-cuttlefish-artifacts android-cuttlefish main" \
+        >/etc/apt/sources.list.d/artifact-registry.list \
+    && apt-get update && apt-get install -y cuttlefish-base cuttlefish-user
+
 WORKDIR /android/
 ENV ANDROID_ARCH=x86_64
 COPY android-install-ndk.sh /android/
 RUN /android/android-install-ndk.sh
 
-# We do not run x86_64-linux-android tests on an android emulator.
-# See ci/android-sysimage.sh for information about how tests are run.
-COPY android-sysimage.sh /android/
-RUN /android/android-sysimage.sh x86_64 x86_64-24_r07.zip
-
 ENV PATH=$PATH:/rust/bin:/android/linux-x86_64/bin \
+    ANDROID_SERIAL=127.0.0.1:6520 \
     CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER=x86_64-linux-android28-clang \
+    CARGO_TARGET_X86_64_LINUX_ANDROID_RUNNER=/tmp/runtest \
     CC_x86_64_linux_android=x86_64-linux-android28-clang \
     CXX_x86_64_linux_android=x86_64-linux-android28-clang++ \
     AR_x86_64_linux_android=llvm-ar \
     HOME=/tmp
+
+ADD runtest-android.rs /tmp/runtest.rs
+COPY cuttlefish-setup.sh docker/x86_64-linux-android/cuttlefish-entrypoint.sh /android/
+ENTRYPOINT ["/android/cuttlefish-entrypoint.sh"]

ci/docker/x86_64-linux-android/cuttlefish-entrypoint.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Boot a Cuttlefish virtual device inside this container, then run the
+# tests against it as an unprivileged user (cvd refuses to run as root).
+#
+# The container needs only /dev/kvm, /dev/net/tun, /dev/vhost-net,
+# /dev/vhost-vsock and CAP_NET_ADMIN, passed by ci/run-docker.sh. It's
+# the same set upstream uses for its own containerized Cuttlefish CI:
+# https://github.com/google/android-cuttlefish/blob/main/.github/workflows/presubmit.yaml
+
+# The device nodes come in with their host-side ownership: open them up
+# for the unprivileged user (container-local, doesn't affect the host).
+chmod 666 /dev/kvm /dev/net/tun /dev/vhost-net /dev/vhost-vsock
+
+# Create the bridges, taps and dnsmasq instances backing the device's
+# virtio-net, inside this container's own network namespace. The deb ships
+# a sysv script for this, usable without systemd, which is also how
+# upstream's container image starts it.
+service cuttlefish-host-resources start
+
+# cvd requires the current process to be in these groups. The groups are
+# normally created by the deb postinst, -f covers any it didn't create.
+for g in kvm cvdnetwork render; do
+    groupadd -f "$g"
+done
+# Match the invoking user's uid so writes to the bind-mounted target/
+# directory stay owned by the host user.
+useradd -m -u "${HOST_UID:?}" -G kvm,cvdnetwork,render cf
+
+run_as_cf() {
+    runuser -u cf -- env PATH="$PATH" HOME=/tmp "$@"
+}
+
+# Keep the device under target/ (bind-mounted from the host) so its logs
+# survive the container for ci/create-artifacts.py to pick up.
+export CUTTLEFISH_DIR=/checkout/target/cuttlefish
+
+run_as_cf /android/cuttlefish-setup.sh
+
+# The test runner (runtest-android.rs) drives the device with the
+# container's adb via $ANDROID_SERIAL. Connect its server to the device
+# (this may replace the server started by the fetched adb during boot).
+run_as_cf adb connect "$ANDROID_SERIAL"
+run_as_cf adb wait-for-device
+
+run_as_cf rustc /tmp/runtest.rs -o /tmp/runtest
+
+rc=0
+run_as_cf "$@" || rc=$?
+
+# Stop the device so its logs are flushed for ci/create-artifacts.py.
+run_as_cf env HOME="$CUTTLEFISH_DIR" \
+    "$CUTTLEFISH_DIR/bin/stop_cvd" -wait_for_launcher=40 || true
+
+exit "$rc"

ci/run-docker.sh

@@ -57,16 +57,37 @@ run() {
     docker build "${build_args[@]}"
 
     mkdir -p target
-    if [ -w /dev/kvm ]; then
-        kvm="--volume /dev/kvm:/dev/kvm"
+
+    extra_args=()
+    if [ -n "${TEST_CUTTLEFISH:-}" ]; then
+        # Cuttlefish-based Android targets boot their virtual device inside
+        # the container (see cuttlefish-entrypoint.sh in the target's docker
+        # dir): pass the virtualization device nodes and let the entrypoint
+        # create its tap networking. It starts as root and drops to an
+        # unprivileged user matching HOST_UID for the device and the tests.
+        # Same flags as upstream's containerized Cuttlefish CI.
+        extra_args+=(
+            --device /dev/kvm
+            --device /dev/net/tun
+            --device /dev/vhost-net
+            --device /dev/vhost-vsock
+            --cap-add NET_ADMIN
+            --security-opt seccomp=unconfined
+            --env CUTTLEFISH_BUILD
+            --env CUTTLEFISH_TARGET
+            --env HOST_UID="$(id -u)"
+        )
     else
-        kvm=""
+        extra_args+=(--user "$(id -u)":"$(id -g)")
+        if [ -w /dev/kvm ]; then
+            extra_args+=(--volume /dev/kvm:/dev/kvm)
+        fi
     fi
 
     docker run \
         --rm \
-        --user "$(id -u)":"$(id -g)" \
         --env LIBC_BUILD_VERBOSE \
+        "${extra_args[@]}" \
         --env LIBC_CI \
         --env LIBC_CI_ZBUILD_STD \
         --env RUSTFLAGS \
@@ -80,7 +101,6 @@ run() {
         --volume "$(rustc --print sysroot)":/rust:ro,Z \
         --volume "$PWD":/checkout:ro,Z \
         --volume "$PWD"/target:/checkout/target \
-        $kvm \
         --init \
         --workdir /checkout \
         "libc-$target" \