Skip to content

Commit 8a2ac51

Browse files
committed
Merge main into codex/linux-headless-platform
2 parents 531ee5d + 173b870 commit 8a2ac51

23 files changed

Lines changed: 940 additions & 46 deletions
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: Linux native SDK
2+
on:
3+
push:
4+
branches: [codex/linux-headless-sdk]
5+
pull_request:
6+
paths:
7+
- 'src/WebScene.NativeWeb/**'
8+
- 'src/WebScene.Sdk/**'
9+
- 'tests/NativeWeb/**'
10+
- 'eng/graphics/**'
11+
- 'eng/sdk/**'
12+
- '.github/workflows/linux-native-sdk.yml'
13+
workflow_dispatch:
14+
permissions:
15+
contents: read
16+
concurrency:
17+
group: linux-native-sdk-${{ github.ref }}
18+
cancel-in-progress: true
19+
jobs:
20+
native:
21+
runs-on: ubuntu-24.04
22+
timeout-minutes: 45
23+
steps:
24+
- uses: actions/checkout@v4
25+
- name: Record source input
26+
run: |
27+
mkdir -p artifacts/linux-native
28+
git rev-parse HEAD > artifacts/linux-native/source-revision.txt
29+
git archive --format=tar.gz HEAD > artifacts/linux-native/source.tar.gz
30+
- name: Install compiler and native prerequisites
31+
run: |
32+
sudo apt-get update
33+
sudo apt-get install -y patchelf ninja-build libfontconfig1-dev libfreetype-dev libvulkan-dev mesa-vulkan-drivers vulkan-tools
34+
python3 -m pip install --break-system-packages cmake==3.31.6
35+
- uses: actions/cache@v4
36+
with:
37+
path: ~/.cache/webscene/llvm-22.1.1
38+
key: linux-cxx-llvm-22.1.1-efc4d945744f951d-v2
39+
- name: Install checksum-pinned LLVM
40+
run: |
41+
python3 eng/sdk/install-linux-llvm.py "$HOME/.cache/webscene/llvm-22.1.1"
42+
echo "WEBSCENE_LLVM_ROOT=$HOME/.cache/webscene/llvm-22.1.1" >> "$GITHUB_ENV"
43+
echo "$HOME/.cache/webscene/llvm-22.1.1/bin" >> "$GITHUB_PATH"
44+
- name: Archive offline compiler profile
45+
run: python3 eng/sdk/archive-linux-llvm.py "$WEBSCENE_LLVM_ROOT" artifacts/linux-native/llvm.tar.gz
46+
- uses: actions/upload-artifact@v4
47+
with:
48+
name: linux-native-compiler
49+
path: artifacts/linux-native/llvm.tar.gz
50+
compression-level: 0
51+
retention-days: 7
52+
- name: Build native document, shared CSS and UI compiler
53+
run: |
54+
cmake -S src/WebScene.NativeWeb -B artifacts/linux-native/build -G Ninja \
55+
-DCMAKE_BUILD_TYPE=Debug \
56+
-DCMAKE_TOOLCHAIN_FILE="$PWD/src/WebScene.Sdk/cmake/WebSceneToolchain.cmake"
57+
cmake --build artifacts/linux-native/build --parallel 2
58+
ctest --test-dir artifacts/linux-native/build --output-on-failure
59+
- name: Install native-only SDK without V8
60+
run: |
61+
cmake -S src/WebScene.Sdk -B artifacts/linux-native/sdk-build -G Ninja \
62+
-DCMAKE_BUILD_TYPE=Release -DWEBSCENE_SDK_WEBGPU=OFF \
63+
-DCMAKE_TOOLCHAIN_FILE="$PWD/src/WebScene.Sdk/cmake/WebSceneToolchain.cmake" \
64+
-DCMAKE_INSTALL_PREFIX="$PWD/artifacts/linux-native/sdk"
65+
cmake --build artifacts/linux-native/sdk-build --parallel 2
66+
cmake --install artifacts/linux-native/sdk-build
67+
python3 eng/sdk/stage-linux-runtime.py --llvm "$WEBSCENE_LLVM_ROOT" --sdk artifacts/linux-native/sdk
68+
tar -czf artifacts/linux-native/sdk.tar.gz -C artifacts/linux-native sdk
69+
- uses: actions/upload-artifact@v4
70+
if: always()
71+
with:
72+
name: linux-native-evidence
73+
path: |
74+
artifacts/linux-native/source.tar.gz
75+
artifacts/linux-native/source-revision.txt
76+
artifacts/linux-native/sdk.tar.gz
77+
artifacts/linux-native/build/Testing/Temporary/*.log
78+
if-no-files-found: error
79+
retention-days: 7
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Linux SDK source contracts
2+
on:
3+
push:
4+
branches: [codex/linux-headless-sdk]
5+
pull_request:
6+
paths: ['eng/sdk/**', 'experiments/WebScene.NativeEngine.Probe/native/graphics/**', 'src/WebScene.Sdk/**', '.github/workflows/linux-sdk-contracts.yml']
7+
permissions:
8+
contents: read
9+
jobs:
10+
contracts:
11+
runs-on: ubuntu-24.04
12+
timeout-minutes: 5
13+
steps:
14+
- uses: actions/checkout@v4
15+
- name: Header scanner regressions and public dependency closure
16+
run: |
17+
python3 eng/sdk/test-header-closure.py
18+
python3 - <<'PY'
19+
import importlib.util
20+
from pathlib import Path
21+
import sys
22+
script=Path('eng/sdk/header-closure.py')
23+
spec=importlib.util.spec_from_file_location('closure',script)
24+
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
25+
root=Path.cwd();native=root/'experiments/WebScene.NativeEngine.Probe/native';authoring=root/'src/WebScene.NativeWeb/include'
26+
pending=[(p,[]) for p in list(authoring.rglob('*.hpp'))+[native/'webscene/compiled_document.hpp',native/'graphics/native_webgpu_surface.h']]
27+
seen=set()
28+
while pending:
29+
item,parents=pending.pop();item=item.resolve()
30+
if item in seen:continue
31+
seen.add(item)
32+
if not any(item.is_relative_to(p) for p in (native,authoring)):
33+
raise RuntimeError('Outside SDK header roots: '+' -> '.join(str(p.relative_to(root)) for p in parents+[item]))
34+
for name in module.includes(item.read_text()):
35+
candidate=next((p for p in [item.parent/name,authoring/name,native/name,native/'graphics'/name] if p.is_file()),None)
36+
if candidate:pending.append((candidate,parents+[item]))
37+
print('Verified',len(seen),'public header dependencies')
38+
module.generate(root)
39+
PY
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Linux native headless SDK profile
2+
3+
The Linux x86_64 SDK is a Native-only producer profile. It installs `WebScene::Core`, `NativeWeb`, `SharedCSS`, `Compiler`, and optionally `WebGPU`. It does not build or package V8, the Runtime component, application scripts or runtime-loaded application HTML. The compiler/ABI pin is LLVM 22.1.1 with libc++. See `src/WebScene.Sdk/cmake/WebSceneLinuxSDK.cmake` and the checksum-pinned installer in `eng/sdk/install-linux-llvm.py`.
4+
5+
HTML/templates are still compiled to C++20 at build time. Applications explicitly select `CSS_BACKEND shared`; native CSS parsing is allowed. The Linux package exports relocatable imported targets and a platform marker so a macOS binary SDK cannot accidentally be consumed on Linux. Requesting unavailable Runtime components fails rather than substituting another host.
6+
7+
## Offscreen WebGPU
8+
9+
On Linux, `native_webgpu_surface` names the reusable `native_headless_webgpu_surface`. The macOS IOSurface and Windows DXGI implementations remain unchanged. The Linux surface uses pinned Dawn Vulkan textures and the existing immutable image-lease/pool contracts. It is an offscreen native authoring/presentation target, **not** a claim of Linux desktop dma-buf/opaque-FD interop, X11/Wayland composition or Avalonia/Uno GPU parity.
10+
11+
Three color allocations are admitted at most, under an explicit byte budget. A retained image remains usable across resize and after surface close; only idle slots can be reclaimed. Producer work retires from actual Dawn queue completion, independently of application frame pumping. Exhausted slots return an empty texture rather than blocking the owner thread or creating an unbounded queue. Adapter identity is exposed, unknown adapters are rejected, and software devices require explicit authorization (`WEBSCENE_HEADLESS_ALLOW_SOFTWARE=1`). CI additionally forces a fallback adapter and records it as software, never hardware qualification.
12+
13+
`capture_native_image` is an explicit native diagnostic/export API. The provider interface does not make CPU-only hosts link Dawn. Captures use bounded GPU-to-buffer copies, wait only on the explicit diagnostic operation, and retain the consumer until GPU copy completion even when mapping times out. Normal frame publication performs no CPU readback. Only top-left 8-bit sRGB images are currently composed by the paired AppScene headless PNG path; unsupported formats fail explicitly.
14+
15+
## Build and validation
16+
17+
```sh
18+
python3 eng/sdk/install-linux-llvm.py "$HOME/.cache/webscene/llvm-22.1.1"
19+
export WEBSCENE_LLVM_ROOT="$HOME/.cache/webscene/llvm-22.1.1"
20+
cmake -S src/WebScene.Sdk -B build-sdk -G Ninja \
21+
-DCMAKE_TOOLCHAIN_FILE="$PWD/src/WebScene.Sdk/cmake/WebSceneToolchain.cmake" \
22+
-DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$PWD/sdk" \
23+
-DWEBSCENE_SDK_WEBGPU=OFF
24+
cmake --build build-sdk --parallel 3
25+
cmake --install build-sdk
26+
python3 eng/sdk/stage-linux-runtime.py --llvm "$WEBSCENE_LLVM_ROOT" --sdk sdk
27+
cmake -S tests/Headless -B build-consumer -G Ninja \
28+
-DCMAKE_PREFIX_PATH="$PWD/sdk" \
29+
-DCMAKE_TOOLCHAIN_FILE="$PWD/sdk/lib/cmake/WebScene/WebSceneToolchain.cmake" \
30+
-DHEADLESS_TEST_WEBGPU=OFF
31+
cmake --build build-consumer --parallel 3
32+
ctest --test-dir build-consumer --output-on-failure
33+
```
34+
35+
For GPU qualification, first build the existing pinned Dawn Linux package with `eng/graphics/build.py dawn --rid linux-x64`, then set `WEBSCENE_SDK_WEBGPU=ON`, `WEBSCENE_GRAPHICS_SDK_ROOT=<graphics-sdk>/linux-x64`, and `HEADLESS_TEST_WEBGPU=ON`. Use the same LLVM/libc++ producer toolchain. Ubuntu 24.04 prerequisites and explicit software-Vulkan configuration are in the paired AppScene workflow.
36+
37+
The installed-consumer tests cover compiled shared-CSS document construction, template identity, native pointer/input interaction, theme layout, bounded frame ownership, idle producer progress, exact captured pixels, retained resize, capture budgets and repeated teardown. They also compile the existing native Kestrel GPU modules against the installed SDK and run the original pipeline, mesh/line pixel, scene invalidation, pending-frame and resize assertions. The only platform change in that original test is Vulkan adapter selection on Linux.
38+
39+
The paired AppScene PR supplies the control host, deterministic stepping, screenshots, native application samples, relocation/bundle auditing and end-to-end tests. This infrastructure does not complete the original JavaScript Kestrel migration. Software-Vulkan results do not establish hardware performance, full conformance, desktop external-memory sharing, media support, or browser/WebGL parity. Issue #46 and the parent graphics epic remain open until their separate acceptance gates are met.

‎eng/sdk/archive-linux-llvm.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env python3
2+
"""Archive only the verified native C++ compiler profile for offline consumers."""
3+
import argparse
4+
import importlib.util
5+
from pathlib import Path
6+
import tarfile
7+
8+
p = argparse.ArgumentParser(description=__doc__)
9+
p.add_argument("source", type=Path)
10+
p.add_argument("output", type=Path)
11+
a = p.parse_args()
12+
spec = importlib.util.spec_from_file_location("installer", Path(__file__).with_name("install-linux-llvm.py"))
13+
module = importlib.util.module_from_spec(spec)
14+
spec.loader.exec_module(module)
15+
if not (a.source / "webscene-toolchain.json").exists():
16+
raise RuntimeError("Compiler provenance marker missing")
17+
a.output.parent.mkdir(parents=True, exist_ok=True)
18+
with tarfile.open(a.output, "w:gz", compresslevel=3) as archive:
19+
archive.add(a.source, arcname="llvm-22.1.1", filter=lambda item: item if module.keep(item) or item.name.endswith("webscene-toolchain.json") else None)

‎eng/sdk/header-closure.py‎

100644100755
Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,50 @@
11
#!/usr/bin/env python3
2-
"""Generate installation rules for public SDK headers and their local includes."""
2+
"""Generate install rules for actual local includes of public SDK headers."""
33
from pathlib import Path
44
import re
55
import sys
6-
root=Path(sys.argv[1]).resolve()
7-
native=root/'experiments/WebScene.NativeEngine.Probe/native'
8-
authoring=root/'src/WebScene.NativeWeb/include'
9-
roots=[authoring,native,native/'graphics']
10-
pending=list(authoring.rglob('*.hpp'))+[native/'webscene/compiled_document.hpp',native/'graphics/native_webgpu_surface.h']
11-
seen=set()
12-
while pending:
13-
item=pending.pop().resolve()
14-
if item in seen:continue
15-
seen.add(item)
16-
for included in re.findall(r'^\s*#\s*include\s*[<"]([^">]+)[">]',item.read_text(),re.M):
17-
candidates=[item.parent/included]+[base/included for base in roots]
18-
match=next((candidate for candidate in candidates if candidate.is_file()),None)
19-
if match:pending.append(match)
20-
elif included.startswith(('webscene_','webscene/')):
21-
raise RuntimeError(f'Unresolved SDK header dependency {included} in {item}')
22-
for item in sorted(seen):
23-
relative=item.relative_to(authoring if item.is_relative_to(authoring) else native)
24-
print(f'install(FILES "{item}" DESTINATION "include/{relative.parent.as_posix()}")')
25-
print(f'set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "{item}")')
6+
7+
TOKENS = re.compile(
8+
r'(?P<include>^[ \t]*\#[ \t]*include[ \t]*[<"](?P<path>[^">\r\n]+)[">])'
9+
r'|(?:u8|u|U|L)?R"(?P<delimiter>[^ ()\\\t\r\n]{0,16})\(.*?\)(?P=delimiter)"'
10+
r'|/\*.*?\*/|//[^\r\n]*'
11+
r'|"(?:\\.|[^"\\\r\n])*"'
12+
r'|(?<![\w])\x27(?:\\.|[^\x27\\\r\n])*\x27',
13+
re.MULTILINE | re.DOTALL,
14+
)
15+
16+
def includes(text):
17+
return [m.group('path') for m in TOKENS.finditer(text) if m.group('include')]
18+
19+
def generate(root):
20+
root=Path(root).resolve()
21+
native=root/'experiments/WebScene.NativeEngine.Probe/native'
22+
authoring=root/'src/WebScene.NativeWeb/include'
23+
roots=[authoring,native,native/'graphics']
24+
pending=[(path,[]) for path in sorted(authoring.rglob('*.hpp'))+[native/'webscene/compiled_document.hpp',native/'graphics/native_webgpu_surface.h']]
25+
seen=set()
26+
while pending:
27+
item,chain=pending.pop();item=item.resolve()
28+
if item in seen:continue
29+
if not any(item.is_relative_to(base) for base in (authoring,native)):
30+
raise RuntimeError('Public header includes a producer-only source: '+' -> '.join(str(path.relative_to(root)) for path in chain+[item]))
31+
seen.add(item)
32+
for included in includes(item.read_text()):
33+
candidates=[item.parent/included]+[base/included for base in roots]
34+
match=next((candidate for candidate in candidates if candidate.is_file()),None)
35+
if match:pending.append((match,chain+[item]))
36+
elif included.startswith(('webscene_','webscene/')):
37+
raise RuntimeError(f'Unresolved SDK header dependency {included} in {item}')
38+
for item in sorted(seen):
39+
relative=item.relative_to(authoring if item.is_relative_to(authoring) else native)
40+
print(f'install(FILES "{item}" DESTINATION "include/{relative.parent.as_posix()}")')
41+
print(f'set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "{item}")')
42+
json_root=root/'samples/NativeKestrel/third_party/nlohmann'
43+
print('if(CMAKE_SYSTEM_NAME STREQUAL "Linux")')
44+
print(f'install(FILES "{json_root}/json.hpp" DESTINATION include/third_party/nlohmann)')
45+
print(f'install(FILES "{json_root}/LICENSE.MIT" DESTINATION share/licenses/WebScene RENAME nlohmann-LICENSE)')
46+
print('endif()')
47+
print(f'WebScene SDK header closure: {len(seen)} files',file=sys.stderr)
48+
49+
if __name__=='__main__':
50+
generate(sys.argv[1])

‎eng/sdk/install-linux-llvm.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env python3
2+
"""Install the checksum-pinned Linux C++ compiler without unrelated LLVM SDKs."""
3+
import argparse
4+
import hashlib
5+
import json
6+
from pathlib import Path, PurePosixPath
7+
import shutil
8+
import subprocess
9+
import tarfile
10+
import tempfile
11+
import urllib.request
12+
13+
VERSION = "22.1.1"
14+
URL = "https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.1/LLVM-22.1.1-Linux-X64.tar.xz"
15+
SHA256 = "efc4d945744f951df00ec72c5b31da5d5a2eaf1d53cc7c9d0644f93f0f9e817d"
16+
BINARIES = {"clang", "clang++", "clang-22", "clang-scan-deps", "llvm-ar", "llvm-ranlib",
17+
"lld", "ld.lld", "llvm-strip", "llvm-nm", "llvm-readobj", "llvm-objdump", "llvm-objcopy"}
18+
19+
def keep(member):
20+
parts = PurePosixPath(member.name).parts[1:]
21+
if not parts:
22+
return True
23+
if parts[0] == "bin":
24+
return len(parts) == 1 or parts[1] in BINARIES
25+
if parts[0] == "include":
26+
return True
27+
if parts[0] == "lib":
28+
return member.isdir() or "clang" in parts or ".so" in parts[-1] or parts[-1].endswith(".ld")
29+
return parts[-1].startswith(("LICENSE", "NOTICE")) or parts[:2] == ("share", "licenses")
30+
31+
def install(destination):
32+
destination = destination.resolve()
33+
stamp = destination / "webscene-toolchain.json"
34+
if stamp.is_file():
35+
if json.loads(stamp.read_text()).get("archiveSha256") != SHA256:
36+
raise RuntimeError("Refusing a different compiler cache")
37+
subprocess.run([destination / "bin/clang++", "--version"], check=True)
38+
return
39+
if destination.exists():
40+
raise RuntimeError("Refusing an incomplete or unrelated installation: " + str(destination))
41+
destination.parent.mkdir(parents=True, exist_ok=True)
42+
with tempfile.TemporaryDirectory(prefix="llvm-install-", dir=destination.parent) as work:
43+
root = Path(work)
44+
archive = root / "llvm.tar.xz"
45+
digest = hashlib.sha256()
46+
with urllib.request.urlopen(URL, timeout=120) as response, archive.open("wb") as output:
47+
while chunk := response.read(1024 * 1024):
48+
digest.update(chunk)
49+
output.write(chunk)
50+
if digest.hexdigest() != SHA256:
51+
raise RuntimeError("LLVM archive checksum mismatch")
52+
extracted = root / "extracted"
53+
extracted.mkdir()
54+
with tarfile.open(archive) as source:
55+
source.extractall(extracted, members=(m for m in source if keep(m)), filter="data")
56+
children = list(extracted.iterdir())
57+
if len(children) != 1 or not (children[0] / "bin/clang++").exists():
58+
raise RuntimeError("Unexpected LLVM archive layout")
59+
subprocess.run([children[0] / "bin/clang++", "--version"], check=True)
60+
shutil.move(str(children[0]), destination)
61+
stamp.write_text(json.dumps({"version": VERSION, "archiveSha256": SHA256, "url": URL,
62+
"profile": "native-cxx"}, indent=2) + "\n")
63+
64+
if __name__ == "__main__":
65+
parser = argparse.ArgumentParser(description=__doc__)
66+
parser.add_argument("destination", type=Path)
67+
install(parser.parse_args().destination)

‎eng/sdk/stage-linux-runtime.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/usr/bin/env python3
2+
"""Copy the pinned C++ shared ABI runtime and its licenses into an installed SDK."""
3+
import argparse
4+
import hashlib
5+
import json
6+
from pathlib import Path
7+
import shutil
8+
import subprocess
9+
10+
11+
def stage(llvm, sdk):
12+
llvm, sdk = llvm.resolve(), sdk.resolve()
13+
destination = sdk / "lib"
14+
destination.mkdir(parents=True, exist_ok=True)
15+
selected = {}
16+
for pattern in ("libc++.so*", "libc++abi.so*", "libunwind.so*"):
17+
candidates = sorted((llvm / "lib").rglob(pattern))
18+
if not candidates:
19+
raise RuntimeError("Missing pinned C++ runtime: " + pattern)
20+
for source in candidates:
21+
# Copy symlink targets too, preserving the SONAME used by ELF consumers.
22+
if not source.is_file():
23+
continue
24+
raw = source.read_bytes()
25+
digest = hashlib.sha256(raw).hexdigest()
26+
if source.name in selected and selected[source.name] != digest:
27+
raise RuntimeError("Ambiguous C++ runtime architecture: " + source.name)
28+
selected[source.name] = digest
29+
target = destination / source.name
30+
shutil.copyfile(source, target, follow_symlinks=True)
31+
if raw.startswith(b"\x7fELF"):
32+
subprocess.run(["patchelf", "--set-rpath", "$ORIGIN", target], check=True)
33+
licenses = sdk / "share/licenses/WebScene/LLVM"
34+
licenses.mkdir(parents=True, exist_ok=True)
35+
for source in llvm.rglob("LICENSE*.TXT"):
36+
shutil.copyfile(source, licenses / ("-".join(source.relative_to(llvm).parts)))
37+
metadata = sdk / "share/webscene"
38+
metadata.mkdir(parents=True, exist_ok=True)
39+
(metadata / "linux-cxx-runtime.json").write_text(json.dumps({
40+
"toolchain": json.loads((llvm / "webscene-toolchain.json").read_text()),
41+
"files": {name: hashlib.sha256((destination / name).read_bytes()).hexdigest() for name in selected},
42+
}, indent=2) + "\n")
43+
44+
if __name__ == "__main__":
45+
p = argparse.ArgumentParser(description=__doc__)
46+
p.add_argument("--llvm", type=Path, required=True)
47+
p.add_argument("--sdk", type=Path, required=True)
48+
a = p.parse_args()
49+
stage(a.llvm, a.sdk)

0 commit comments

Comments
 (0)