diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0543beefb39c..2656d082d1743 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -236,6 +236,9 @@ jobs: - target: x86_64-apple-darwin os: macos-26-intel - target: x86_64-linux-android + env: { TEST_CUTTLEFISH: 1 } + # Keep in sync with the Android build pinned in ci/cuttlefish-setup.sh + artifact-tag: android17 # FIXME: Exec format error (os error 8) # - target: x86_64-unknown-linux-gnux32 - target: x86_64-unknown-linux-musl diff --git a/ci/README.md b/ci/README.md index 4e5b552d58892..54017ccc0d6eb 100644 --- a/ci/README.md +++ b/ci/README.md @@ -31,8 +31,12 @@ builds are run on stable/beta/nightly, but are the only ones that do so. The remaining architectures look like: -* Android runs in a [docker image][android-docker] with an emulator, the NDK, - and the SDK already set up. The entire build happens within the docker image. +* Android x86_64 tests run on a [Cuttlefish](https://source.android.com/docs/devices/cuttlefish) + virtual device booted on the CI host by `cuttlefish-setup.sh`; the + [docker image][android-docker] cross-compiles the tests with the NDK and + drives the device over adb. The remaining Android targets run in docker + images with the legacy SDK emulator, the NDK, and the SDK already set up, + with the entire build happening within the docker image. * The MIPS, ARM, and AArch64 builds all use the QEMU userspace emulator to run the generated binary to actually verify the tests pass. * The MUSL build just has to download a MUSL compiler and target libraries and diff --git a/ci/android-sysimage.sh b/ci/android-sysimage.sh deleted file mode 100755 index 98484b3521f60..0000000000000 --- a/ci/android-sysimage.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -URL=https://dl.google.com/android/repository/sys-img/android - -main() { - local arch="${1}" - local name="${2}" - local dest=/system - local td - td="$(mktemp -d)" - - apt-get install --no-install-recommends e2tools - - pushd "${td}" - wget -q --tries=5 "${URL}/${name}" - unzip -q "${name}" - - local system - system="$(find . -name system.img)" - mkdir -p ${dest}/{bin,lib,lib64} - - # Extract android linker and libraries to /system - # This allows android executables to be run directly (or with qemu) - if [ "${arch}" = "x86_64" ] || [ "${arch}" = "arm64" ]; then - e2cp -p "${system}:/bin/linker64" "${dest}/bin/" - e2cp -p "${system}:/lib64/libdl.so" "${dest}/lib64/" - e2cp -p "${system}:/lib64/libc.so" "${dest}/lib64/" - e2cp -p "${system}:/lib64/libm.so" "${dest}/lib64/" - else - e2cp -p "${system}:/bin/linker" "${dest}/bin/" - e2cp -p "${system}:/lib/libdl.so" "${dest}/lib/" - e2cp -p "${system}:/lib/libc.so" "${dest}/lib/" - e2cp -p "${system}:/lib/libm.so" "${dest}/lib/" - fi - - # clean up - apt-get purge --auto-remove -y e2tools - - popd - - rm -rf "${td}" -} - -main "${@}" diff --git a/ci/create-artifacts.py b/ci/create-artifacts.py index 4145e20491e24..2104b9cac3837 100755 --- a/ci/create-artifacts.py +++ b/ci/create-artifacts.py @@ -32,11 +32,20 @@ def main(): archive_name = f"archive-{now}" archive_path = f"{archive_name}.tar.gz" - sp.run( - ["tar", "czvf", archive_path, "-C", build_dir, "-T-"], - input=file_list, - check=True, - ) + # --dereference because kernel.log is a symlink into logs/ + tar_cmd = ["tar", "czvf", archive_path, "--dereference", "-C", build_dir, "-T-"] + + # Grab the runtime logs of the Cuttlefish device launched by ci/cuttlefish-setup.sh. + # The device runs inside the test container, which keeps it under the bind-mounted + # target/ so the logs are visible here. + cf_dir = Path(os.getenv("CUTTLEFISH_DIR", "target/cuttlefish")).resolve() + cf_runtime = cf_dir / "cuttlefish" / "instances" / "cvd-1" + cf_logs = ["logs/logcat", "logs/launcher.log", "kernel.log"] + cf_found = [log for log in cf_logs if cf_runtime.joinpath(log).exists()] + if cf_found: + tar_cmd += ["-C", str(cf_runtime), *cf_found] + + sp.run(tar_cmd, input=file_list, check=True) # If we are in GHA, set these env vars for future use gh_env = os.getenv("GITHUB_ENV") diff --git a/ci/cuttlefish-setup.sh b/ci/cuttlefish-setup.sh new file mode 100755 index 0000000000000..14bb782d40887 --- /dev/null +++ b/ci/cuttlefish-setup.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -eux + +# Fetch and boot a Cuttlefish Android virtual device, heavily inspired by +# Mesa's CI: +# https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/.gitlab-ci/cuttlefish-runner.sh +# +# Runs unprivileged (cvd refuses to run as root) inside the test container, +# invoked by cuttlefish-entrypoint.sh after it has prepared the devices, +# networking and groups (see ci/docker/x86_64-linux-android/). +build="${CUTTLEFISH_BUILD:-15660610}" +target="${CUTTLEFISH_TARGET:-aosp_cf_x86_64_only_phone-userdebug}" +cf_dir="${CUTTLEFISH_DIR:-$HOME/cuttlefish}" + +mkdir -p "$cf_dir" +cvd fetch --default_build="${build}/${target}" --target_directory="$cf_dir" + +cd "$cf_dir" + +# -daemon exits only once the guest has fully booted. +# -enable_sandbox=false is needed because crosvm's minijail device sandbox +# (auto-enabled when /var/empty exists) requires unprivileged user +# namespaces, which Ubuntu 24.04+ blocks via AppArmor by default. +HOME="$cf_dir" timeout 10m ./bin/launch_cvd \ + -daemon \ + -enable_sandbox=false \ + -verbosity=INFO \ + -file_verbosity=DEBUG \ + -enable_audio=false \ + -enable_bootanimation=false \ + -enable_minimal_mode=true \ + -enable_modem_simulator=false \ + -enable_wifi=false \ + -report_anonymous_usage_stats=no \ + -cpus=2 \ + -memory_mb=4096 + +# Check the guest's adbd is reachable on the port the test runner uses, +# and log the Android version actually being tested. +HOME="$cf_dir" ./bin/adb connect 127.0.0.1:6520 +HOME="$cf_dir" ./bin/adb -s 127.0.0.1:6520 wait-for-device +HOME="$cf_dir" ./bin/adb devices +HOME="$cf_dir" ./bin/adb -s 127.0.0.1:6520 shell \ + getprop ro.build.version.release diff --git a/ci/docker/x86_64-linux-android/Dockerfile b/ci/docker/x86_64-linux-android/Dockerfile index bc9580e6e1c0d..62eee968057bc 100644 --- a/ci/docker/x86_64-linux-android/Dockerfile +++ b/ci/docker/x86_64-linux-android/Dockerfile @@ -1,26 +1,38 @@ FROM ubuntu:26.04 RUN apt-get update && apt-get install -y --no-install-recommends \ + adb \ ca-certificates \ + curl \ gcc \ libc-dev \ python3 \ unzip \ wget +# Cuttlefish host packages: the virtual device the tests run on boots +# inside this container (see cuttlefish-entrypoint.sh for the docker run +# flags this needs). +RUN curl -fsSL https://us-apt.pkg.dev/doc/repo-signing-key.gpg \ + -o /etc/apt/trusted.gpg.d/artifact-registry.asc \ + && echo "deb https://us-apt.pkg.dev/projects/android-cuttlefish-artifacts android-cuttlefish main" \ + >/etc/apt/sources.list.d/artifact-registry.list \ + && apt-get update && apt-get install -y cuttlefish-base cuttlefish-user + WORKDIR /android/ ENV ANDROID_ARCH=x86_64 COPY android-install-ndk.sh /android/ RUN /android/android-install-ndk.sh -# We do not run x86_64-linux-android tests on an android emulator. -# See ci/android-sysimage.sh for information about how tests are run. -COPY android-sysimage.sh /android/ -RUN /android/android-sysimage.sh x86_64 x86_64-24_r07.zip - ENV PATH=$PATH:/rust/bin:/android/linux-x86_64/bin \ + ANDROID_SERIAL=127.0.0.1:6520 \ CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER=x86_64-linux-android28-clang \ + CARGO_TARGET_X86_64_LINUX_ANDROID_RUNNER=/tmp/runtest \ CC_x86_64_linux_android=x86_64-linux-android28-clang \ CXX_x86_64_linux_android=x86_64-linux-android28-clang++ \ AR_x86_64_linux_android=llvm-ar \ HOME=/tmp + +ADD runtest-android.rs /tmp/runtest.rs +COPY cuttlefish-setup.sh docker/x86_64-linux-android/cuttlefish-entrypoint.sh /android/ +ENTRYPOINT ["/android/cuttlefish-entrypoint.sh"] diff --git a/ci/docker/x86_64-linux-android/cuttlefish-entrypoint.sh b/ci/docker/x86_64-linux-android/cuttlefish-entrypoint.sh new file mode 100755 index 0000000000000..42c99ae8fa58c --- /dev/null +++ b/ci/docker/x86_64-linux-android/cuttlefish-entrypoint.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -eux + +# Boot a Cuttlefish virtual device inside this container, then run the +# tests against it as an unprivileged user (cvd refuses to run as root). +# +# The container needs only /dev/kvm, /dev/net/tun, /dev/vhost-net, +# /dev/vhost-vsock and CAP_NET_ADMIN, passed by ci/run-docker.sh. It's +# the same set upstream uses for its own containerized Cuttlefish CI: +# https://github.com/google/android-cuttlefish/blob/main/.github/workflows/presubmit.yaml + +# The device nodes come in with their host-side ownership: open them up +# for the unprivileged user (container-local, doesn't affect the host). +chmod 666 /dev/kvm /dev/net/tun /dev/vhost-net /dev/vhost-vsock + +# Create the bridges, taps and dnsmasq instances backing the device's +# virtio-net, inside this container's own network namespace. The deb ships +# a sysv script for this, usable without systemd, which is also how +# upstream's container image starts it. +service cuttlefish-host-resources start + +# cvd requires the current process to be in these groups. The groups are +# normally created by the deb postinst, -f covers any it didn't create. +for g in kvm cvdnetwork render; do + groupadd -f "$g" +done +# Match the invoking user's uid so writes to the bind-mounted target/ +# directory stay owned by the host user. +useradd -m -u "${HOST_UID:?}" -G kvm,cvdnetwork,render cf + +run_as_cf() { + runuser -u cf -- env PATH="$PATH" HOME=/tmp "$@" +} + +# Keep the device under target/ (bind-mounted from the host) so its logs +# survive the container for ci/create-artifacts.py to pick up. +export CUTTLEFISH_DIR=/checkout/target/cuttlefish + +run_as_cf /android/cuttlefish-setup.sh + +# The test runner (runtest-android.rs) drives the device with the +# container's adb via $ANDROID_SERIAL. Connect its server to the device +# (this may replace the server started by the fetched adb during boot). +run_as_cf adb connect "$ANDROID_SERIAL" +run_as_cf adb wait-for-device + +run_as_cf rustc /tmp/runtest.rs -o /tmp/runtest + +rc=0 +run_as_cf "$@" || rc=$? + +# Stop the device so its logs are flushed for ci/create-artifacts.py. +run_as_cf env HOME="$CUTTLEFISH_DIR" \ + "$CUTTLEFISH_DIR/bin/stop_cvd" -wait_for_launcher=40 || true + +exit "$rc" diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 2266b2f2d2690..699cd4099b3d4 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -57,16 +57,37 @@ run() { docker build "${build_args[@]}" mkdir -p target - if [ -w /dev/kvm ]; then - kvm="--volume /dev/kvm:/dev/kvm" + + extra_args=() + if [ -n "${TEST_CUTTLEFISH:-}" ]; then + # Cuttlefish-based Android targets boot their virtual device inside + # the container (see cuttlefish-entrypoint.sh in the target's docker + # dir): pass the virtualization device nodes and let the entrypoint + # create its tap networking. It starts as root and drops to an + # unprivileged user matching HOST_UID for the device and the tests. + # Same flags as upstream's containerized Cuttlefish CI. + extra_args+=( + --device /dev/kvm + --device /dev/net/tun + --device /dev/vhost-net + --device /dev/vhost-vsock + --cap-add NET_ADMIN + --security-opt seccomp=unconfined + --env CUTTLEFISH_BUILD + --env CUTTLEFISH_TARGET + --env HOST_UID="$(id -u)" + ) else - kvm="" + extra_args+=(--user "$(id -u)":"$(id -g)") + if [ -w /dev/kvm ]; then + extra_args+=(--volume /dev/kvm:/dev/kvm) + fi fi docker run \ --rm \ - --user "$(id -u)":"$(id -g)" \ --env LIBC_BUILD_VERBOSE \ + "${extra_args[@]}" \ --env LIBC_CI \ --env LIBC_CI_ZBUILD_STD \ --env RUSTFLAGS \ @@ -80,7 +101,6 @@ run() { --volume "$(rustc --print sysroot)":/rust:ro,Z \ --volume "$PWD":/checkout:ro,Z \ --volume "$PWD"/target:/checkout/target \ - $kvm \ --init \ --workdir /checkout \ "libc-$target" \