Skip to content

test(ffe): use canonical fixtures #1979

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leoromanovsky wants to merge 7 commits into
base: main
Choose a base branch
from
Open

test(ffe): use canonical fixtures #1979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
99892b0
Use canonical FFE fixtures
leoromanovsky May 12, 2026
9faafd3
Update FFE fixture submodule
leoromanovsky May 12, 2026
393938a
Fix canonical FFE fixture CI paths
leoromanovsky May 13, 2026
2c1a5e0
Resolve FFE fixture paths from package root
leoromanovsky May 13, 2026
247a5e0
test(ffe): stop asserting fixture reasons
leoromanovsky May 13, 2026
ef516b4
test(ffe): update fixtures submodule to main
leoromanovsky May 13, 2026
4aefcc6
ci(benchmarks): initialize submodules
leoromanovsky May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
.dockerignore @DataDog/libdatadog-core
.github/ @DataDog/apm-common-components-core
.gitignore @DataDog/libdatadog
.gitmodules @DataDog/libdatadog
.gitlab-ci.yml @DataDog/apm-common-components-core
.gitlab/benchmarks.yml @DataDog/apm-common-components-core
.gitlab/fuzz.yml @DataDog/chaos-platform
Expand Down
6 changes: 6 additions & 0 deletions .github/dependabot.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
version: 2
updates:
- package-ecosystem: "gitsubmodule"
directory: "/"
schedule:
interval: "weekly"
4 changes: 4 additions & 0 deletions .github/workflows/test-ffi.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ jobs:
steps:
- name: Checkout sources
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
with:
submodules: recursive
- name: Setup output dir
shell: bash
run: |
Expand Down Expand Up @@ -224,6 +226,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
with:
submodules: recursive
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # 3.10.0
with:
Expand Down
23 changes: 23 additions & 0 deletions .gitlab/benchmarks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,29 @@ benchmarks:
- export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :)
- git clone --branch libdatadog/benchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform /platform && cd /platform
- ./steps/capture-hardware-software-info.sh
- |
SCRIPT_DIR="$(pwd)/steps"
source ./steps/config-benchmark-env.sh

checkout_source_with_submodules() {
local repo_url=$1
local dest=$2
local target=$3

if [[ -d "${dest}" ]]; then
return
fi

git clone "${repo_url}" "${dest}"
git -C "${dest}" checkout "${target}"
git -C "${dest}" submodule update --init --recursive
}

checkout_source_with_submodules "${UPSTREAM_REPO_URL}" "${CANDIDATE_PATH}" "${CANDIDATE_IDENTIFIER}"

# Refresh benchmark-platform's derived baseline after candidate checkout.
VERBOSE="true" source ./steps/config-benchmark-env.sh
checkout_source_with_submodules "${UPSTREAM_REPO_URL}" "${BASELINE_PATH}" "${BASELINE_IDENTIFIER}"
- ./steps/run-benchmarks.sh
- ./steps/analyze-results.sh
- "./steps/upload-results-to-s3.sh || :"
Expand Down
3 changes: 3 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "datadog-ffe/ffe-system-test-data"]
path = datadog-ffe/ffe-system-test-data
url = https://github.com/DataDog/ffe-system-test-data.git
13 changes: 11 additions & 2 deletions datadog-ffe/benches/eval.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,17 @@ use datadog_ffe::rules_based::{
UniversalFlagConfig,
};

const UFC_CONFIG_PATH: &str = concat!(
env!("CARGO_MANIFEST_DIR"),
"/ffe-system-test-data/ufc-config.json"
);
const EVALUATION_CASES_DIR: &str = concat!(
env!("CARGO_MANIFEST_DIR"),
"/ffe-system-test-data/evaluation-cases"
);

fn load_configuration_bytes() -> Vec<u8> {
fs::read("tests/data/flags-v1.json").expect("Failed to read flags-v1.json")
fs::read(UFC_CONFIG_PATH).expect("Failed to read ufc-config.json")
}

#[derive(Debug, Serialize, Deserialize)]
Expand All @@ -33,7 +42,7 @@ struct TestResult {
fn load_test_cases() -> Vec<TestCase> {
let mut test_cases = Vec::new();

if let Ok(entries) = fs::read_dir("tests/data/tests") {
if let Ok(entries) = fs::read_dir(EVALUATION_CASES_DIR) {
for entry in entries.flatten() {
if let Some(path_str) = entry.path().to_str() {
if path_str.ends_with(".json") {
Expand Down
1 change: 1 addition & 0 deletions datadog-ffe/ffe-system-test-data
Open in desktop
Copy link
Copy Markdown
Contributor

@hoolioh hoolioh May 14, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since datadog-ffe is used in several SDKs we would eventually need to publish it. Having git submodules embedded into the crate proved to be problematic for third party consumers. In order to minimize risks, I would recommend the following options:

  1. Create another crate for the canonical tests which will contain the git submodule and pulls the datadog-ffe as a dependency, this approach is used in another pojects like serde (test-suite). That way there will be no dependency to an internal repo on the published crate. That crate can be part of the workspace, the tests will be run each time datadog-ffe is modified and since it will be mark with publish = false there will be no risk of getting into trouble with downstream projects.
  2. If option 1 is not feasible because of time of any project limitation, at the very least I would consider feature gate those tests and disable them by default. That way third party consumers won't fall into building issues if the tests are triggered. However this will require to modify the CI a bit in order to include that feature into the tests workflow.

Submodule ffe-system-test-data added at 444637
18 changes: 14 additions & 4 deletions datadog-ffe/src/rules_based/eval/eval_assignment.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,13 +237,23 @@ mod tests {
fn evaluation_sdk_test_data() {
let _ = env_logger::builder().is_test(true).try_init();

let config =
UniversalFlagConfig::from_json(std::fs::read("tests/data/flags-v1.json").unwrap())
.unwrap();
let config = UniversalFlagConfig::from_json(
std::fs::read(concat!(
env!("CARGO_MANIFEST_DIR"),
"/ffe-system-test-data/ufc-config.json"
))
.unwrap(),
)
.unwrap();
let config = Configuration::from_server_response(config);
let now = Utc::now();

for entry in fs::read_dir("tests/data/tests/").unwrap() {
for entry in fs::read_dir(concat!(
env!("CARGO_MANIFEST_DIR"),
"/ffe-system-test-data/evaluation-cases/"
))
.unwrap()
{
let entry = entry.unwrap();
println!("Processing test file: {:?}", entry.path());

Expand Down
6 changes: 5 additions & 1 deletion datadog-ffe/src/rules_based/ufc/models.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,11 @@ mod tests {
#[test]
#[cfg_attr(miri, ignore)] // this test is way too slow on miri
fn parse_flags_v1() {
let json_content = std::fs::read_to_string("tests/data/flags-v1.json").unwrap();
let json_content = std::fs::read_to_string(concat!(
env!("CARGO_MANIFEST_DIR"),
"/ffe-system-test-data/ufc-config.json"
))
.unwrap();
let ufc: UniversalFlagConfigWire = serde_json::from_str(&json_content).unwrap();

let failures = ufc
Expand Down
Loading
Loading