diff --git a/assets/validate/README.md b/assets/validate/README.md index 09b575e..7a2ebd7 100644 --- a/assets/validate/README.md +++ b/assets/validate/README.md @@ -7,5 +7,17 @@ Default **`ASSETS_ROOT`**: XSLT 1.0 stylesheets for IVOA registry rules beyond X | `checkIVOAOAI.xsl` | IVOA harvest profile tests on OAI GET responses (phase 2) | | `checkVOResource.xsl` | VOResource constraint tests on harvested records (phase 3) | | `validationCommon.xsl` | Shared helpers (imported by the check stylesheets) | +| `validateVocabularies.xsl` | Generated vocabulary-term checks (imported by `checkVOResource.xsl`) | +| `vocabularyControlled.csv` | Config for which VOResource paths are vocabulary-controlled | + +## Regenerating vocabulary XSLT + +IVOA vocabularies change at most a few times a year. When they do (or when `vocabularyControlled.csv` changes), regenerate the committed stylesheet and commit the result: + +```bash +benson generate-vocabulary-xsl +``` + +This fetches the vocabularies listed in `vocabularyControlled.csv` from `http://www.ivoa.net/rdf/` and overwrites `validateVocabularies.xsl`. Do not edit that XSLT by hand. Full context: [`docs/schemas-and-validation-assets.md`](../../docs/schemas-and-validation-assets.md). diff --git a/docs/schemas-and-validation-assets.md b/docs/schemas-and-validation-assets.md index e12ca50..95c6462 100644 --- a/docs/schemas-and-validation-assets.md +++ b/docs/schemas-and-validation-assets.md @@ -137,11 +137,23 @@ These stylesheets implement IVOA registry **business rules** that XSD alone does | `checkIVOAOAI.xsl` | Phase 2 | Profile tests on OAI GET responses (`Identify`, `ListMetadataFormats`, `ListSets`, `ListRecords`). Emits `` elements. | | `checkVOResource.xsl` | Phase 3 | Additional constraints on harvested VOResource records. Emits ``. | | `validationCommon.xsl` | (imported) | Shared helpers for the check stylesheets | +| `validateVocabularies.xsl` | Phase 3 (imported) | Generated checks for vocabulary-controlled VOResource / VODataService fields | +| `vocabularyControlled.csv` | (generator input) | Paths and IVOA vocabulary URLs used to build `validateVocabularies.xsl` | If XSLT processing fails or the stylesheet is missing, phase 2/3 fall back to simpler pass/fail heuristics (HTTP status and absence of OAI error codes). **Note:** `checkIVOAOAI.xsl` references `testsVOResource-v1_0.xsl` from the legacy Java tree; that file is **not** shipped under `assets/validate/`. Import failures are caught and the code falls back to non-XSLT checks. +### Regenerating `validateVocabularies.xsl` + +The vocabulary stylesheet is generated offline and committed (not at server start), so standalone validation uses cached terms. When IVOA vocabularies or `vocabularyControlled.csv` change: + +```bash +benson generate-vocabulary-xsl +``` + +Commit the updated `validateVocabularies.xsl`. Implementation: [`src/benson/oai/vocabulary_xslt.py`](../src/benson/oai/vocabulary_xslt.py). + --- ## `assets/standards/` — IVOA standard records (not XSDs) diff --git a/src/benson/main.py b/src/benson/main.py index fa67610..e53742c 100644 --- a/src/benson/main.py +++ b/src/benson/main.py @@ -151,10 +151,25 @@ async def run() -> int: print(f"Searchables cache refreshed ({count} registries).") +def _run_generate_vocabulary_xsl(args: argparse.Namespace) -> None: + from benson.config import Settings + from benson.oai.vocabulary_xslt import generate + + settings = Settings.from_env() + out_path = generate(settings.assets_root) + print(f"Wrote {out_path}") + + def main() -> None: if len(sys.argv) == 1 or ( len(sys.argv) > 1 - and sys.argv[1] not in ("serve", "check-publishers", "sync-searchables") + and sys.argv[1] + not in ( + "serve", + "check-publishers", + "sync-searchables", + "generate-vocabulary-xsl", + ) ): sys.argv.insert(1, "serve") @@ -196,6 +211,11 @@ def main() -> None: help="Print result as JSON", ) + subparsers.add_parser( + "generate-vocabulary-xsl", + help="Regenerate assets/validate/validateVocabularies.xsl from IVOA vocabularies", + ) + args = parser.parse_args() if args.command in (None, "serve"): _run_serve(args) @@ -203,6 +223,8 @@ def main() -> None: _run_check_publishers(args) elif args.command == "sync-searchables": _run_sync_searchables(args) + elif args.command == "generate-vocabulary-xsl": + _run_generate_vocabulary_xsl(args) else: parser.error(f"unknown command: {args.command}") diff --git a/src/benson/oai/vocabulary_xslt.py b/src/benson/oai/vocabulary_xslt.py index 077c615..659634b 100644 --- a/src/benson/oai/vocabulary_xslt.py +++ b/src/benson/oai/vocabulary_xslt.py @@ -4,14 +4,12 @@ against IVOA vocabularies. This produces the file assets/validate/validateVocabularies.xsl and -is supposed to be executed in validate whenever registry-relevant -vocabularies change. The current plan is that in a checkout of benson, -you do:: +is supposed to be executed whenever registry-relevant vocabularies +change (a few times a year at most). From a checkout of benson:: - cd assets/validate - python ../../src/benson/oai/vocabulary_xslt.py + benson generate-vocabulary-xsl -Then git commit your changes and re-deploy. +Then git commit the updated XSLT and re-deploy. [MD: this is fast enough that we could generate the XSLT at server start; but when people do stand-alone validation, I think I'd rather @@ -24,6 +22,7 @@ import functools import json import os +from pathlib import Path from urllib import request IVOA_VOCABULARY_ROOT = "http://www.ivoa.net/rdf/" @@ -149,27 +148,36 @@ def make_validator_for(xpath, attpath, voc_name, mandatory): voc_url="http://www.ivoa.net/rdf/"+voc_name) -def read_config(): +def read_config(validate_dir): # returns a list of tuples as per CONFIG_FILE. # we should probably do some input validation, too. import csv - with open(CONFIG_FILE, newline="") as config_source: + config_path = Path(validate_dir) / CONFIG_FILE + with open(config_path, newline="") as config_source: return [tup for tup in csv.reader(config_source, skipinitialspace=True)] -def main(): - conf = read_config() +def generate(validate_dir): + """Fetch vocabularies and write validateVocabularies.xsl under validate_dir.""" + validate_dir = Path(validate_dir) + conf = read_config(validate_dir) fragments = [] for vocspec in conf: fragments.append( make_validator_for(*vocspec)) - with open("validateVocabularies.xsl", "w") as f: + out_path = validate_dir / "validateVocabularies.xsl" + with open(out_path, "w") as f: f.write((XSL_BODY.format( "\n\n".join(fragments)))) + return out_path + + +def main(): + generate(".") if __name__=="__main__":