diff --git a/LICENSE b/LICENSE index 7d8cf8e89..4b4793d6b 100644 --- a/LICENSE +++ b/LICENSE @@ -202,7 +202,7 @@ ------------------------------- -examples/deep-researcher prompts.py and utils.py are copied from +examples/deep-researcher prompts.py and deep_researcher_utils.py are copied from https://github.com/langchain-ai/local-deep-researcher and are licensed under the MIT License. MIT License diff --git a/LICENSE-wheel b/LICENSE-wheel index b1c8c03b0..476a56a5b 100644 --- a/LICENSE-wheel +++ b/LICENSE-wheel @@ -202,7 +202,7 @@ ------------------------------- -examples/deep-researcher prompts.py and utils.py are copied from +examples/deep-researcher prompts.py and deep_researcher_utils.py are copied from https://github.com/langchain-ai/local-deep-researcher and are licensed under the MIT License. MIT License diff --git a/burr-redirect/build.sh b/burr-redirect/build.sh index 8f078dbb5..d805d61ac 100755 --- a/burr-redirect/build.sh +++ b/burr-redirect/build.sh @@ -33,10 +33,18 @@ fi VERSION="$1" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# Requires an activated virtualenv with the release tooling installed: +# pip install -e ".[developer]" (provides build and twine) +if [ -z "${VIRTUAL_ENV:-}" ]; then + echo "ERROR: no active Python virtualenv detected. Activate the release venv first." >&2 + exit 1 +fi + echo "Building burr redirect package for version ${VERSION}..." -# Stamp version into pyproject.toml from template -sed "s/VERSION/${VERSION}/g" "${SCRIPT_DIR}/pyproject.toml.template" > "${SCRIPT_DIR}/pyproject.toml" +# Generate pyproject.toml from the template, stamping in the version and the +# extras declared in the repository root pyproject.toml +python "${SCRIPT_DIR}/generate_pyproject.py" "${VERSION}" # Clean previous build rm -rf "${SCRIPT_DIR}/dist" "${SCRIPT_DIR}/build" "${SCRIPT_DIR}"/*.egg-info diff --git a/burr-redirect/generate_pyproject.py b/burr-redirect/generate_pyproject.py new file mode 100644 index 000000000..23ac60239 --- /dev/null +++ b/burr-redirect/generate_pyproject.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Generate burr-redirect/pyproject.toml from the template and the root pyproject. + +The redirect package (PyPI name ``burr``) exists only to point users at +``apache-burr``. Every extra that ``apache-burr`` declares must also exist on +the redirect package, otherwise ``pip install burr[langfuse]`` warns about an +unknown extra and silently installs nothing. Hand-mirroring the extras drifted +in the past, so the ``[project.optional-dependencies]`` section is generated +here from the extras declared in the repository root ``pyproject.toml``. + +Usage: + python generate_pyproject.py + python generate_pyproject.py 0.42.0 +""" + +import sys +from pathlib import Path + +try: + import tomllib +except ImportError: # Python < 3.11 + import tomli as tomllib + +PLACEHOLDER = "#OPTIONAL_DEPENDENCIES#" + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent +ROOT_PYPROJECT = REPO_ROOT / "pyproject.toml" +TEMPLATE = SCRIPT_DIR / "pyproject.toml.template" +OUTPUT = SCRIPT_DIR / "pyproject.toml" + + +def fail(message: str) -> None: + """Print an error and exit nonzero.""" + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def load_toml(path: Path) -> dict: + """Parse a TOML file, failing with a clear message if it is unreadable.""" + try: + with open(path, "rb") as f: + return tomllib.load(f) + except FileNotFoundError: + fail(f"{path} not found") + except tomllib.TOMLDecodeError as e: + fail(f"{path} is not valid TOML: {e}") + + +def read_root_extras() -> list: + """Return the alphabetically sorted extra names declared by apache-burr.""" + extras = load_toml(ROOT_PYPROJECT).get("project", {}).get("optional-dependencies") + if not extras: + fail(f"no [project.optional-dependencies] found in {ROOT_PYPROJECT}") + return sorted(extras) + + +def render_extras(extras: list, version: str) -> str: + """Render the [project.optional-dependencies] section for the redirect package.""" + lines = ["[project.optional-dependencies]"] + lines += [f'{name} = ["apache-burr[{name}]=={version}"]' for name in extras] + return "\n".join(lines) + + +def render_template(extras: list, version: str) -> str: + """Substitute the extras section and the version into the template text.""" + template = TEMPLATE.read_text() + matches = [line for line in template.splitlines() if PLACEHOLDER in line] + if not matches: + fail( + f"placeholder {PLACEHOLDER} not found in {TEMPLATE}. " + "The template must contain that line so the generated " + "[project.optional-dependencies] section has somewhere to go." + ) + rendered = "\n".join( + render_extras(extras, version) if PLACEHOLDER in line else line + for line in template.splitlines() + ) + if not rendered.endswith("\n"): + rendered += "\n" + return rendered.replace("VERSION", version) + + +def verify(extras: list, version: str) -> None: + """Re-parse the written file and assert it says exactly what we intended.""" + project = load_toml(OUTPUT).get("project", {}) + + written_extras = project.get("optional-dependencies", {}) + if sorted(written_extras) != extras: + missing = sorted(set(extras) - set(written_extras)) + unexpected = sorted(set(written_extras) - set(extras)) + fail( + f"{OUTPUT} extras do not match {ROOT_PYPROJECT}: " + f"missing={missing} unexpected={unexpected}" + ) + + for name in extras: + expected = [f"apache-burr[{name}]=={version}"] + if written_extras[name] != expected: + fail(f"{OUTPUT} extra {name!r} is {written_extras[name]!r}, expected {expected!r}") + + expected_deps = [f"apache-burr=={version}"] + if project.get("dependencies") != expected_deps: + fail( + f"{OUTPUT} dependencies are {project.get('dependencies')!r}, expected {expected_deps!r}" + ) + + if project.get("version") != version: + fail(f"{OUTPUT} version is {project.get('version')!r}, expected {version!r}") + + +def main(argv: list) -> None: + if len(argv) != 2: + print(f"Usage: {Path(argv[0]).name} ", file=sys.stderr) + print(f"Example: {Path(argv[0]).name} 0.42.0", file=sys.stderr) + sys.exit(1) + + version = argv[1] + extras = read_root_extras() + OUTPUT.write_text(render_template(extras, version)) + verify(extras, version) + print(f"Wrote {OUTPUT} for version {version} with {len(extras)} extras.") + + +if __name__ == "__main__": + main(sys.argv) diff --git a/burr-redirect/pyproject.toml.template b/burr-redirect/pyproject.toml.template index cb52080b0..6c101a786 100644 --- a/burr-redirect/pyproject.toml.template +++ b/burr-redirect/pyproject.toml.template @@ -19,6 +19,13 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" +# This package ships no code at all -- it only declares a dependency on +# apache-burr. Without this, setuptools flat-layout auto-discovery would pick up +# generate_pyproject.py sitting next to this file and install it as a top-level +# module into every user's site-packages. +[tool.setuptools] +py-modules = [] + [project] name = "burr" version = "VERSION" @@ -28,35 +35,10 @@ requires-python = ">=3.9" license = "Apache-2.0" dependencies = ["apache-burr==VERSION"] -[project.optional-dependencies] -aiosqlite = ["apache-burr[aiosqlite]==VERSION"] -asyncpg = ["apache-burr[asyncpg]==VERSION"] -bedrock = ["apache-burr[bedrock]==VERSION"] -cli = ["apache-burr[cli]==VERSION"] -developer = ["apache-burr[developer]==VERSION"] -documentation = ["apache-burr[documentation]==VERSION"] -examples = ["apache-burr[examples]==VERSION"] -graphviz = ["apache-burr[graphviz]==VERSION"] -hamilton = ["apache-burr[hamilton]==VERSION"] -haystack = ["apache-burr[haystack]==VERSION"] -inappexamples = ["apache-burr[inappexamples]==VERSION"] -learn = ["apache-burr[learn]==VERSION"] -opentelemetry = ["apache-burr[opentelemetry]==VERSION"] -postgresql = ["apache-burr[postgresql]==VERSION"] -psycopg2 = ["apache-burr[psycopg2]==VERSION"] -pydantic = ["apache-burr[pydantic]==VERSION"] -pymongo = ["apache-burr[pymongo]==VERSION"] -ray = ["apache-burr[ray]==VERSION"] -redis = ["apache-burr[redis]==VERSION"] -release = ["apache-burr[release]==VERSION"] -start = ["apache-burr[start]==VERSION"] -streamlit = ["apache-burr[streamlit]==VERSION"] -tests = ["apache-burr[tests]==VERSION"] -tracking = ["apache-burr[tracking]==VERSION"] -tracking-client = ["apache-burr[tracking-client]==VERSION"] -tracking-client-s3 = ["apache-burr[tracking-client-s3]==VERSION"] -tracking-server = ["apache-burr[tracking-server]==VERSION"] -tracking-server-s3 = ["apache-burr[tracking-server-s3]==VERSION"] +# The optional-dependencies section is generated by generate_pyproject.py from +# the extras declared in the repository root pyproject.toml, so the redirect +# package never drifts out of sync with apache-burr. Do not list extras here. +#OPTIONAL_DEPENDENCIES# [project.urls] Homepage = "https://burr.apache.org/" diff --git a/pyproject.toml b/pyproject.toml index 6a17ee988..8ce1bf8a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -216,6 +216,7 @@ developer = [ "build", "twine", "pre-commit", + "tomli; python_version < '3.11'", ] opentelemetry = [ @@ -289,6 +290,7 @@ exclude = [ ".git/**", ".github/**", "docs/**", + "website/**", # Exclude VCS and system files ".gitignore", ".gitmodules", @@ -301,6 +303,7 @@ exclude = [ # NOTE: If you add/remove examples, update this list AND tests/test_release_config.py "examples/README.md", "examples/validate_examples.py", + "examples/fastapi_mount_example.py", "examples/adaptive-crag/**", "examples/conversational-rag/**", "examples/custom-serde/**", diff --git a/scripts/check_asf_headers.py b/scripts/check_asf_headers.py index 15d44a2e5..b20e3227c 100644 --- a/scripts/check_asf_headers.py +++ b/scripts/check_asf_headers.py @@ -106,7 +106,7 @@ def main(argv: Optional[list] = None) -> int: if not files: return 0 - repo_root = _find_repo_root(files[0].parent) + repo_root = _find_repo_root(Path(__file__).resolve().parent) patterns = _load_rat_exclude_patterns(repo_root) violations = [] diff --git a/tests/test_release_config.py b/tests/test_release_config.py index 7d2f82cf6..5c5c71b8d 100644 --- a/tests/test_release_config.py +++ b/tests/test_release_config.py @@ -70,13 +70,18 @@ def test_examples_include_exclude_coverage(): include_patterns = flit_sdist.get("include", []) exclude_patterns = flit_sdist.get("exclude", []) - # Extract example directories from include patterns + # Extract example directories and files from include patterns included_examples = set() + included_files = set() for pattern in include_patterns: - if pattern.startswith("examples/") and pattern.endswith("/**"): - # Extract directory name from patterns like "examples/email-assistant/**" - dir_name = pattern.removeprefix("examples/").removesuffix("/**") - included_examples.add(dir_name) + if pattern.startswith("examples/"): + if pattern.endswith("/**"): + # Extract directory name from patterns like "examples/email-assistant/**" + dir_name = pattern.removeprefix("examples/").removesuffix("/**") + included_examples.add(dir_name) + else: + # File pattern like "examples/__init__.py" + included_files.add(pattern.removeprefix("examples/")) # Extract example directories from exclude patterns excluded_examples = set() @@ -111,6 +116,10 @@ def test_examples_include_exclude_coverage(): missing_from_config = actual_dirs - configured_dirs extra_in_config = configured_dirs - actual_dirs + configured_files = included_files | excluded_files + files_missing_from_config = actual_files - configured_files + extra_files_in_config = configured_files - actual_files + # Build error message if mismatch found errors = [] @@ -140,15 +149,78 @@ def test_examples_include_exclude_coverage(): f"\n To fix: Remove these entries from pyproject.toml [tool.flit.sdist]\n" ) + if files_missing_from_config: + errors.append( + f"\nāŒ Top-level files in examples/ that are NOT in pyproject.toml config:\n" + f" {sorted(files_missing_from_config)}\n" + f"\n WHY THIS MATTERS:\n" + f" Loose files directly under examples/ are picked up by flit's package\n" + f" auto-discovery just like the subdirectories, so they ship in the sdist as\n" + f" stray files unless explicitly excluded (this is how\n" + f" examples/fastapi_mount_example.py leaked into a release artifact).\n" + f"\n To fix: Add to pyproject.toml [tool.flit.sdist]:\n" + f" - To INCLUDE in Apache release: add 'examples/' to 'include' list\n" + f" - To EXCLUDE from Apache release: add 'examples/' to 'exclude' list\n" + ) + + if extra_files_in_config: + errors.append( + f"\nāŒ Files listed in pyproject.toml but NOT in examples/ on disk:\n" + f" {sorted(extra_files_in_config)}\n" + f"\n To fix: Remove these entries from pyproject.toml [tool.flit.sdist]\n" + ) + # Report what's currently configured (for debugging) if errors: summary = ( f"\nšŸ“‹ Current configuration:\n" f" Included examples ({len(included_examples)}): {sorted(included_examples)}\n" f" Excluded examples ({len(excluded_examples)}): {sorted(excluded_examples)}\n" + f" Included files ({len(included_files)}): {sorted(included_files)}\n" f" Excluded files ({len(excluded_files)}): {sorted(excluded_files)}\n" f" Actual directories ({len(actual_dirs)}): {sorted(actual_dirs)}\n" + f" Actual files ({len(actual_files)}): {sorted(actual_files)}\n" ) errors.append(summary) assert not errors, "\n".join(errors) + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="tomllib requires Python 3.11+") +def test_non_source_trees_excluded_from_sdist(): + """ + Verify that top-level trees which are not "source used to build" are excluded from + the sdist. + + WHY THIS TEST EXISTS: + scripts/README.md defines the policy table for what belongs in each artifact. The + docs/ and website/ trees are included in the git archive (tar.gz) for voters to + review, but must NOT appear in the sdist or the wheel since they are not needed to + build or use the package. website/ previously shipped in the sdist because it had no + exclude entry. + + If this test fails, add the missing pattern to [tool.flit.sdist] exclude. + """ + project_root = Path(__file__).parent.parent + pyproject_path = project_root / "pyproject.toml" + + with open(pyproject_path, "rb") as f: + config = tomllib.load(f) + + exclude_patterns = set(config["tool"]["flit"]["sdist"]["exclude"]) + + # Trees that exist in the repo but must never be part of the sdist. + non_source_trees = ["docs", "website", "burr-redirect"] + + missing = [ + f"{tree}/**" + for tree in non_source_trees + if (project_root / tree).is_dir() and f"{tree}/**" not in exclude_patterns + ] + + assert not missing, ( + f"\nāŒ Non-source trees missing from [tool.flit.sdist] exclude: {missing}\n" + f"\n These directories are not source needed to build or use the package (see\n" + f" the policy table in scripts/README.md) and would otherwise ship in the sdist.\n" + f"\n To fix: add the listed pattern(s) to pyproject.toml [tool.flit.sdist] exclude.\n" + )