ci: Run aarch64-linux-android tests with qemu-user

.github/workflows/ci.yaml

@@ -196,6 +196,8 @@ jobs:
       matrix:
         include:
           - target: aarch64-linux-android
+            # Keep in sync with the Android build pinned in ci/android-bionic-sysroot.sh
+            artifact-tag: android17
           - target: aarch64-unknown-linux-musl
           - target: aarch64-unknown-linux-musl
             env: { TEST_MUSL_V1_2_3: 1 }
@@ -236,6 +238,9 @@ jobs:
           - target: x86_64-apple-darwin
             os: macos-26-intel
           - target: x86_64-linux-android
+            env: { TEST_CUTTLEFISH: 1 }
+            # Keep in sync with the Android build pinned in ci/cuttlefish-setup.sh
+            artifact-tag: android17
           # FIXME: Exec format error (os error 8)
           # - target: x86_64-unknown-linux-gnux32
           - target: x86_64-unknown-linux-musl

ci/README.md

@@ -31,8 +31,15 @@ builds are run on stable/beta/nightly, but are the only ones that do so.
 
 The remaining architectures look like:
 
-* Android runs in a [docker image][android-docker] with an emulator, the NDK,
-  and the SDK already set up. The entire build happens within the docker image.
+* Android x86_64 tests run on a [Cuttlefish](https://source.android.com/docs/devices/cuttlefish)
+  virtual device booted on the CI host by `cuttlefish-setup.sh`; the
+  [docker image][android-docker] cross-compiles the tests with the NDK and
+  drives the device over adb. Android aarch64 tests run under the QEMU
+  userspace emulator against the bionic linker and libraries extracted from
+  a current Android build by `android-bionic-sysroot.sh`, with no Android
+  system booted. The remaining Android target (32-bit arm) runs in a docker
+  image with the legacy SDK emulator, the NDK, and the SDK already set up,
+  with the entire build happening within the docker image.
 * The MIPS, ARM, and AArch64 builds all use the QEMU userspace emulator to run
   the generated binary to actually verify the tests pass.
 * The MUSL build just has to download a MUSL compiler and target libraries and

ci/android-bionic-sysroot.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Extract the bionic linker and libraries from a current Android build into
+# /system, so the test binaries can run under qemu-user with no Android
+# system booted (see the aarch64-linux-android Dockerfile). This is a
+# stopgap until GitHub's arm64 runners expose KVM, at which point this
+# target can move to Cuttlefish like x86_64-linux-android.
+#
+# The target_files zip is used because it contains the system partition as
+# plain files, unlike the flashable image zips.
+#
+# The pinned build (currently Android 17) matches ci/cuttlefish-setup.sh;
+# bump both together with the android* artifact-tags in ci.yaml.
+
+build="${ANDROID_SYSROOT_BUILD:-15660610}"
+target="${ANDROID_SYSROOT_TARGET:-aosp_cf_arm64_only_phone-userdebug}"
+
+zip_name="${target%-userdebug}-target_files-${build}.zip"
+url="https://ci.android.com/builds/submitted/${build}/${target}/latest/${zip_name}"
+
+td="$(mktemp -d)"
+
+# The ci.android.com URL serves an HTML page embedding a short-lived signed
+# Cloud Storage URL (JSON-escaped '&'); scrape it to get the artifact.
+signed_url="$(wget -q -O - "${url}" |
+    grep -o 'https://storage.googleapis.com/android-build/[^"]*' |
+    head -1 |
+    sed 's/\\u0026/\&/g')"
+wget -q --tries=5 -O "${td}/target_files.zip" "${signed_url}"
+
+# Take the full lib64 rather than cherry-picking libc/libm/libdl: the
+# linker pulls in supporting libraries that vary between releases.
+# Extraction is best-effort; the sanity checks below decide.
+unzip -q "${td}/target_files.zip" \
+    'SYSTEM/apex/com.android.runtime.apex' \
+    'SYSTEM/bin/linker64' \
+    'SYSTEM/bin/bootstrap/*' \
+    'SYSTEM/build.prop' \
+    'SYSTEM/etc/ld.config*' \
+    'SYSTEM/lib64/*' \
+    -d "${td}" || true
+
+mkdir -p /system
+mv "${td}/SYSTEM/bin" /system/bin
+mv "${td}/SYSTEM/lib64" /system/lib64
+mv "${td}/SYSTEM/build.prop" /system/build.prop
+if [ -d "${td}/SYSTEM/etc" ]; then
+    mv "${td}/SYSTEM/etc" /system/etc
+fi
+
+# Since Android 10 bionic lives in the Runtime APEX and the /system paths
+# are symlinks into /apex/com.android.runtime; unpack the apex payload
+# there so they resolve.
+unzip -q "${td}/SYSTEM/apex/com.android.runtime.apex" apex_payload.img \
+    -d "${td}"
+mkdir -p /apex/com.android.runtime
+case "$(file -b "${td}/apex_payload.img")" in
+    *EROFS*)
+        fsck.erofs "--extract=/apex/com.android.runtime" \
+            "${td}/apex_payload.img"
+        ;;
+    *ext[24]*)
+        debugfs -R "rdump / /apex/com.android.runtime" \
+            "${td}/apex_payload.img"
+        ;;
+    *)
+        echo "unrecognized apex payload filesystem" >&2
+        exit 1
+        ;;
+esac
+
+rm -rf "${td}"
+
+# Sanity-check the files the tests need (following the /apex symlinks, so
+# this also verifies the apex extraction) and log the Android version.
+test -s /system/bin/linker64
+test -s /system/lib64/libc.so
+test -s /system/lib64/libm.so
+test -s /system/lib64/libdl.so
+grep ro.build.version.release /system/build.prop

ci/create-artifacts.py

@@ -32,11 +32,20 @@ def main():
     archive_name = f"archive-{now}"
     archive_path = f"{archive_name}.tar.gz"
 
-    sp.run(
-        ["tar", "czvf", archive_path, "-C", build_dir, "-T-"],
-        input=file_list,
-        check=True,
-    )
+    # --dereference because kernel.log is a symlink into logs/
+    tar_cmd = ["tar", "czvf", archive_path, "--dereference", "-C", build_dir, "-T-"]
+
+    # Grab the runtime logs of the Cuttlefish device launched by ci/cuttlefish-setup.sh.
+    # The device runs inside the test container, which keeps it under the bind-mounted
+    # target/ so the logs are visible here.
+    cf_dir = Path(os.getenv("CUTTLEFISH_DIR", "target/cuttlefish")).resolve()
+    cf_runtime = cf_dir / "cuttlefish" / "instances" / "cvd-1"
+    cf_logs = ["logs/logcat", "logs/launcher.log", "kernel.log"]
+    cf_found = [log for log in cf_logs if cf_runtime.joinpath(log).exists()]
+    if cf_found:
+        tar_cmd += ["-C", str(cf_runtime), *cf_found]
+
+    sp.run(tar_cmd, input=file_list, check=True)
 
     # If we are in GHA, set these env vars for future use
     gh_env = os.getenv("GITHUB_ENV")

ci/cuttlefish-setup.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Fetch and boot a Cuttlefish Android virtual device, heavily inspired by
+# Mesa's CI:
+# https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/.gitlab-ci/cuttlefish-runner.sh
+#
+# Runs unprivileged (cvd refuses to run as root) inside the test container,
+# invoked by cuttlefish-entrypoint.sh after it has prepared the devices,
+# networking and groups (see ci/docker/x86_64-linux-android/).
+build="${CUTTLEFISH_BUILD:-15660610}"
+target="${CUTTLEFISH_TARGET:-aosp_cf_x86_64_only_phone-userdebug}"
+cf_dir="${CUTTLEFISH_DIR:-$HOME/cuttlefish}"
+
+mkdir -p "$cf_dir"
+cvd fetch --default_build="${build}/${target}" --target_directory="$cf_dir"
+
+cd "$cf_dir"
+
+# -daemon exits only once the guest has fully booted.
+# -enable_sandbox=false is needed because crosvm's minijail device sandbox
+#  (auto-enabled when /var/empty exists) requires unprivileged user
+#  namespaces, which Ubuntu 24.04+ blocks via AppArmor by default.
+HOME="$cf_dir" timeout 10m ./bin/launch_cvd \
+    -daemon \
+    -enable_sandbox=false \
+    -verbosity=INFO \
+    -file_verbosity=DEBUG \
+    -enable_audio=false \
+    -enable_bootanimation=false \
+    -enable_minimal_mode=true \
+    -enable_modem_simulator=false \
+    -enable_wifi=false \
+    -report_anonymous_usage_stats=no \
+    -cpus=2 \
+    -memory_mb=4096
+
+# Check the guest's adbd is reachable on the port the test runner uses,
+# and log the Android version actually being tested.
+HOME="$cf_dir" ./bin/adb connect 127.0.0.1:6520
+HOME="$cf_dir" ./bin/adb -s 127.0.0.1:6520 wait-for-device
+HOME="$cf_dir" ./bin/adb devices
+HOME="$cf_dir" ./bin/adb -s 127.0.0.1:6520 shell \
+    getprop ro.build.version.release

ci/docker/aarch64-linux-android/Dockerfile

@@ -1,46 +1,29 @@
 FROM ubuntu:26.04
 
-RUN dpkg --add-architecture i386
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
-    expect \
+    e2fsprogs \
+    erofs-utils \
     file \
     gcc \
-    libc6-dev \
-    libpulse0 \
-    libstdc++6:i386 \
-    openjdk-8-jre \
+    libc-dev \
     python3 \
+    qemu-user \
     unzip \
     wget
 
 WORKDIR /android/
-COPY android* /android/
-
-ENV ANDROID_ARCH=aarch64
-ENV PATH=$PATH:/android/linux-x86_64/bin:/android/sdk/cmdline-tools/tools:/android/sdk/platform-tools
-
+COPY android-install-ndk.sh android-bionic-sysroot.sh /android/
 RUN /android/android-install-ndk.sh
-RUN /android/android-install-sdk.sh $ANDROID_ARCH
-RUN mv /root/.android /tmp
-RUN chmod 777 -R /tmp/.android
-RUN chmod 755 /android/sdk/cmdline-tools/tools/* /android/sdk/emulator/qemu/linux-x86_64/*
+RUN /android/android-bionic-sysroot.sh
 
-ENV PATH=$PATH:/rust/bin \
-    CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=aarch64-linux-android28-clang \
-    CARGO_TARGET_AARCH64_LINUX_ANDROID_RUNNER=/tmp/runtest \
-    CC_aarch64_linux_android=aarch64-linux-android28-clang \
+# Test binaries run under qemu-user against the bionic extracted to
+# /system by android-bionic-sysroot.sh. This is a stopgap until GitHub's
+# arm64 runners expose KVM, at which point we can move to Cuttlefish like
+# the x86_64-linux-android test.
+ENV PATH=$PATH:/rust/bin:/android/linux-x86_64/bin \
+    CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=aarch64-linux-android24-clang \
+    CARGO_TARGET_AARCH64_LINUX_ANDROID_RUNNER=qemu-aarch64 \
+    CC_aarch64_linux_android=aarch64-linux-android24-clang \
     AR_aarch64_linux_android=llvm-ar \
     HOME=/tmp
-
-ADD runtest-android.rs /tmp/runtest.rs
-ENTRYPOINT [ \
-    "bash", \
-    "-c", \
-    # set SHELL so android can detect a 64bits system, see
-    # http://stackoverflow.com/a/41789144
-    "SHELL=/bin/dash /android/sdk/emulator/emulator @aarch64 -no-window & \
-     rustc /tmp/runtest.rs -o /tmp/runtest && \
-     exec \"$@\"", \
-    "--" \
-]

ci/docker/arm-linux-androideabi/Dockerfile

@@ -27,9 +27,9 @@ RUN chmod 777 -R /tmp/.android
 RUN chmod 755 /android/sdk/cmdline-tools/tools/* /android/sdk/emulator/qemu/linux-x86_64/*
 
 ENV PATH=$PATH:/rust/bin \
-    CARGO_TARGET_ARM_LINUX_ANDROIDEABI_LINKER=armv7a-linux-androideabi28-clang \
+    CARGO_TARGET_ARM_LINUX_ANDROIDEABI_LINKER=armv7a-linux-androideabi24-clang \
     CARGO_TARGET_ARM_LINUX_ANDROIDEABI_RUNNER=/tmp/runtest \
-    CC_arm_linux_androideabi=armv7a-linux-androideabi28-clang \
+    CC_arm_linux_androideabi=armv7a-linux-androideabi24-clang \
     AR_arm_linux_androideabi=llvm-ar \
     HOME=/tmp
 

ci/docker/x86_64-linux-android/Dockerfile

@@ -1,26 +1,38 @@
 FROM ubuntu:26.04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    adb \
     ca-certificates \
+    curl \
     gcc \
     libc-dev \
     python3 \
     unzip \
     wget
 
+# Cuttlefish host packages: the virtual device the tests run on boots
+# inside this container (see cuttlefish-entrypoint.sh for the docker run
+# flags this needs).
+RUN curl -fsSL https://us-apt.pkg.dev/doc/repo-signing-key.gpg \
+        -o /etc/apt/trusted.gpg.d/artifact-registry.asc \
+    && echo "deb https://us-apt.pkg.dev/projects/android-cuttlefish-artifacts android-cuttlefish main" \
+        >/etc/apt/sources.list.d/artifact-registry.list \
+    && apt-get update && apt-get install -y cuttlefish-base cuttlefish-user
+
 WORKDIR /android/
 ENV ANDROID_ARCH=x86_64
 COPY android-install-ndk.sh /android/
 RUN /android/android-install-ndk.sh
 
-# We do not run x86_64-linux-android tests on an android emulator.
-# See ci/android-sysimage.sh for information about how tests are run.
-COPY android-sysimage.sh /android/
-RUN /android/android-sysimage.sh x86_64 x86_64-24_r07.zip
-
 ENV PATH=$PATH:/rust/bin:/android/linux-x86_64/bin \
-    CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER=x86_64-linux-android28-clang \
-    CC_x86_64_linux_android=x86_64-linux-android28-clang \
-    CXX_x86_64_linux_android=x86_64-linux-android28-clang++ \
+    ANDROID_SERIAL=127.0.0.1:6520 \
+    CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER=x86_64-linux-android24-clang \
+    CARGO_TARGET_X86_64_LINUX_ANDROID_RUNNER=/tmp/runtest \
+    CC_x86_64_linux_android=x86_64-linux-android24-clang \
+    CXX_x86_64_linux_android=x86_64-linux-android24-clang++ \
     AR_x86_64_linux_android=llvm-ar \
     HOME=/tmp
+
+ADD runtest-android.rs /tmp/runtest.rs
+COPY cuttlefish-setup.sh docker/x86_64-linux-android/cuttlefish-entrypoint.sh /android/
+ENTRYPOINT ["/android/cuttlefish-entrypoint.sh"]