vey-dcgen: refactor

.github/workflows/codecov.yml

@@ -208,6 +208,42 @@ jobs:
           name: vey-iploc-all-test
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: false
+  vey-dcgen-test:
+    name: vey-dcgen test
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v7
+        with:
+          submodules: true
+      - name: Install rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v2
+        with:
+          toolchain: stable
+          components: llvm-tools
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install capnproto libjemalloc-dev libssl-dev pkg-config clang libbpf-dev
+      - name: Install binutils
+        run: |
+          cargo install cargo-binutils@0.3.6
+      - name: Cargo clean
+        run: cargo clean
+      - name: run unit test
+        run: |
+          ./scripts/coverage/vey-dcgen.sh
+
+      - name: Upload coverage data
+        uses: codecov/codecov-action@v7
+        with:
+          fail_ci_if_error: true
+          disable_search: true
+          files: output.lcov
+          flags: vey-dcgen
+          name: vey-dcgen-all-test
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: false
   vey-proxy-test:
     name: vey-proxy test
     runs-on: ubuntu-26.04

.github/workflows/sphinx.yml

@@ -21,6 +21,7 @@ jobs:
           - vey-gateway
           - vey-keyless
           - vey-iploc
+          - vey-dcgen
           - vey-statsd
     steps:
       - uses: actions/checkout@v7

Cargo.toml

@@ -2,6 +2,8 @@
 members = [
     "vey-bench",
     "vey-dcgen",
+    "vey-dcgen/proto",
+    "vey-dcgen/utils/ctl",
     "vey-iploc",
     "vey-iploc/proto",
     "vey-iploc/utils/ctl",
@@ -75,6 +77,8 @@ members = [
 default-members = [
     "vey-bench",
     "vey-dcgen",
+    "vey-dcgen/proto",
+    "vey-dcgen/utils/ctl",
     "vey-iploc",
     "vey-iploc/proto",
     "vey-iploc/utils/ctl",

scripts/coverage/vey-dcgen.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+set -e
+
+SCRIPTS_DIR=$(dirname "$0")
+PROJECT_DIR=$(realpath "${SCRIPTS_DIR}/../..")
+
+
+TEST_NAME="vey-dcgen-ci"
+. "${SCRIPTS_DIR}/enter.sh"
+
+# build
+FEATURES="jemalloc"
+if command -v clang >/dev/null 2>&1 && pkg-config --exists libbpf; then
+	FEATURES="${FEATURES},ebpf"
+fi
+cargo build --features "${FEATURES}" -p vey-dcgen -p vey-dcgen-ctl
+
+all_binaries=$(find target/debug/ -maxdepth 1 -type f -perm /111 | awk '{print "-object "$0}')
+all_objects=$(find target/debug/deps/ -type f -perm /111 -not -name "*.so" | awk '{print "-object "$0}')
+
+# run vey-dcgen tests
+
+cargo test --features "${FEATURES}" -p vey-dcgen -p vey-dcgen-ctl
+
+RUN_DIR="${PROJECT_DIR}/scripts/testcases/vey-dcgen"
+. "${RUN_DIR}/run.sh"
+
+# get all profraw files generated in each test
+profraw_files=$(find . -type f -regex ".*/${TEST_NAME}.*\.profraw")
+
+# get indexed profile data file
+cargo profdata -- merge -o "${PROF_DATA_FILE}" ${profraw_files}
+
+# report to console
+
+IGNORE_FLAGS="--ignore-filename-regex=.cargo \
+    --ignore-filename-regex=rustc \
+    --ignore-filename-regex=target/debug/build \
+    --ignore-filename-regex=vey-bench \
+    --ignore-filename-regex=vey-iploc \
+    --ignore-filename-regex=vey-mkcert \
+    --ignore-filename-regex=vey-proxy \
+    --ignore-filename-regex=vey-gateway \
+    --ignore-filename-regex=vey-keyless \
+    --ignore-filename-regex=vey-statsd"
+
+echo "==== Coverage for all ===="
+cargo cov -- report --use-color --instr-profile="${PROF_DATA_FILE}" ${IGNORE_FLAGS} ${all_binaries} ${all_objects}
+cargo cov -- export --format=lcov --instr-profile="${PROF_DATA_FILE}" ${IGNORE_FLAGS} ${all_binaries} ${all_objects} > output.lcov

scripts/testcases/vey-dcgen/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc

scripts/testcases/vey-dcgen/main.yaml

@@ -0,0 +1,25 @@
+---
+
+runtime:
+  task_wait_delay: 0s
+  task_wait_timeout: 5s
+  task_quit_timeout: 2s
+
+worker:
+  thread_number: 2
+
+stat:
+  target:
+    udp: 127.0.0.1:8125
+
+server:
+  - name: local
+    type: udp
+    listen: 127.0.0.1:2999
+  - name: local_unix
+    type: unix
+    listen: /tmp/vey-dcgen-ci.sock
+
+issuer:
+  ca_certificate: ../../../vey-dcgen/examples/simple/G3-test.crt
+  ca_private_key: ../../../vey-dcgen/examples/simple/G3-test.key

scripts/testcases/vey-dcgen/query.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+"""Send one certificate-request datagram and wait for a reply.
+
+The payload is a MessagePack fixstr of the hostname. Exit 0 when a reply
+contains a PEM certificate, 2 on timeout.
+"""
+
+import os
+import socket
+import sys
+
+
+def encode_host(host: str) -> bytes:
+    raw = host.encode()
+    if len(raw) > 31:
+        raise SystemExit(f"host {host} is too long for a msgpack fixstr")
+    return bytes([0xA0 | len(raw)]) + raw
+
+
+def main() -> None:
+    if len(sys.argv) != 4:
+        raise SystemExit("usage: query.py udp|unix TARGET HOST|--invalid")
+
+    kind, target, host = sys.argv[1:]
+    payload = b"not a request" if host == "--invalid" else encode_host(host)
+
+    if kind == "udp":
+        host_addr, port = target.rsplit(":", 1)
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.settimeout(2.0)
+        sock.sendto(payload, (host_addr, int(port)))
+    elif kind == "unix":
+        client = target + ".client"
+        try:
+            os.unlink(client)
+        except FileNotFoundError:
+            pass
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+        sock.bind(client)
+        sock.settimeout(2.0)
+        try:
+            sock.sendto(payload, target)
+        except OSError:
+            os.unlink(client)
+            raise
+    else:
+        raise SystemExit(f"unknown transport {kind}")
+
+    try:
+        data, _addr = sock.recvfrom(65535)
+    except (TimeoutError, socket.timeout):
+        raise SystemExit(2)
+    finally:
+        sock.close()
+        if kind == "unix":
+            try:
+                os.unlink(target + ".client")
+            except FileNotFoundError:
+                pass
+
+    if b"BEGIN CERTIFICATE" not in data:
+        raise SystemExit("reply did not contain a certificate")
+
+
+if __name__ == "__main__":
+    main()

scripts/testcases/vey-dcgen/run.sh

@@ -0,0 +1,109 @@
+# Binaries default to a coverage build. PGO points BIN_DIR at a directory
+# that contains the instrumented daemon and the control client.
+BIN_DIR=${BIN_DIR:-${PROJECT_DIR}/target/debug}
+
+UNIX_SOCK=/tmp/vey-dcgen-ci.sock
+
+cleanup()
+{
+	if [ -n "${DCGEN_PID:-}" ]; then
+		kill -INT "$DCGEN_PID" 2>/dev/null || true
+		wait "$DCGEN_PID" 2>/dev/null || true
+	fi
+	rm -f "${UNIX_SOCK}" "${UNIX_SOCK}.client"
+	if command -v clear_profiles >/dev/null 2>&1; then
+		clear_profiles
+	fi
+	if [ -n "${PGO_BIN_DIR:-}" ]; then
+		rm -rf "${PGO_BIN_DIR}"
+	fi
+	if [ -n "${PGO_RAW_DIR:-}" ]; then
+		rm -rf "${PGO_RAW_DIR}"
+	fi
+}
+
+trap cleanup EXIT
+
+vey_dcgen_ctl()
+{
+	"${BIN_DIR}"/vey-dcgen-ctl -G ${TEST_NAME} -p $DCGEN_PID "$@"
+}
+
+query_udp()
+{
+	python3 "${RUN_DIR}/query.py" udp 127.0.0.1:2999 "$1"
+}
+
+query_unix()
+{
+	python3 "${RUN_DIR}/query.py" unix "${UNIX_SOCK}" "$1"
+}
+
+expect_datagram_miss()
+{
+	if "$@"; then
+		echo "expected no datagram reply: $*" >&2
+		exit 1
+	fi
+}
+
+wait_until_ready()
+{
+	i=0
+	while [ "$i" -lt 25 ]; do
+		if ! kill -0 "$DCGEN_PID" 2>/dev/null; then
+			echo "vey-dcgen exited before the control socket was ready" >&2
+			wait "$DCGEN_PID" || true
+			DCGEN_PID=
+			exit 1
+		fi
+		if vey_dcgen_ctl version >/dev/null; then
+			return
+		fi
+		i=$((i + 1))
+		sleep 0.2
+	done
+	echo "timed out waiting for vey-dcgen" >&2
+	exit 1
+}
+
+rm -f "${UNIX_SOCK}" "${UNIX_SOCK}.client"
+
+set -x
+
+"${BIN_DIR}"/vey-dcgen -Vvv
+
+for dir in $(ls "${PROJECT_DIR}/vey-dcgen/examples")
+do
+	example_dir="${PROJECT_DIR}/vey-dcgen/examples/${dir}"
+	[ -d "${example_dir}" ] || continue
+
+	"${BIN_DIR}"/vey-dcgen -c "${example_dir}" -t
+done
+
+"${BIN_DIR}"/vey-dcgen -c "${RUN_DIR}" -t
+
+"${BIN_DIR}"/vey-dcgen -c "${RUN_DIR}" -G ${TEST_NAME} &
+DCGEN_PID=$!
+
+wait_until_ready
+
+query_udp www.example.com
+query_unix www.example.com
+expect_datagram_miss query_udp --invalid
+expect_datagram_miss query_unix --invalid
+
+vey_dcgen_ctl reload
+query_udp www.example.com
+
+vey_dcgen_ctl reload-issuer
+query_udp www.example.com
+
+# Let one stats emit interval pass before shutdown.
+sleep 0.5
+
+vey_dcgen_ctl offline
+wait "$DCGEN_PID"
+DCGEN_PID=
+
+set +x