Rename

.github/ISSUE_TEMPLATE/new_dataset.yaml

@@ -1,16 +1,15 @@
 name: "New Dataset"
-description: Add a new dataset to the EM Database
+description: Add a new dataset or model weights to the EM Database
 title: "[New Dataset]: "
 labels:
   - "new_dataset"
-assignees:
-  - CSSFrancis
 body:
   - type: markdown
     attributes:
       value: |
-        Thanks for adding a new dataset to the EM Database! Please fill out the following
-        form regarding the dataset you are submitting. Make sure to include as much relevant
+        Thanks for adding a new entry to the EM Database! Please fill out the following
+        form regarding the dataset or model weights you are submitting. Make sure to
+        include as much relevant metadata as you can.
   - type: input
     id: dataset_name
     attributes:
@@ -32,17 +31,33 @@ body:
     attributes:
       label: --Affiliation--
       description: Which institution or company are you at?
-      placeholder: ex. University of Wisconsin-Madison
+      placeholder: ex. University of Mars
     validations:
       required: true
+  - type: input
+    id: orcid
+    attributes:
+      label: --ORCID--
+      description: Your ORCID, as 0000-0000-0000-0000.
+      placeholder: ex. 0000-0000-0000-0000
+    validations:
+      required: false
   - type: input
     id: url
     attributes:
       label: --URL--
-      description: Please provide a URL where the dataset can be downloaded. This must be a direct link to the file itself - the file name is taken from the end of it.
+      description: Please provide a URL where the file can be downloaded. A link ending in the file name is split into the directory it is served from and the name; a link that names no file - a Google Drive `uc?export=download&id=` link, or anything else with a query string - is kept whole, and then the file name has to be given below.
       placeholder: ex. https://zenodo.org/records/15490547/files/smallPtychography.hspy
     validations:
       required: true
+  - type: input
+    id: file_name
+    attributes:
+      label: --File Name--
+      description: Only needed when the URL above does not end in the file name. This is the name the file is saved under locally.
+      placeholder: ex. smallPtychography.hspy
+    validations:
+      required: false
   - type: input
     id: checksum
     attributes:
@@ -91,6 +106,14 @@ body:
       placeholder: ex. Gen 1 Titan
     validations:
       required: false
+  - type: input
+    id: camera_length
+    attributes:
+      label: Camera Length
+      description: Please provide the camera length, with its unit.
+      placeholder: ex. 100 mm
+    validations:
+      required: false
   - type: input
     id: accelerating_voltage
     attributes:
@@ -121,11 +144,65 @@ body:
         - In-situ TEM
         - Cryo-EM
         - Other
+  - type: input
+    id: doi
+    attributes:
+      label: DOI
+      description: DOI of the record or the paper, without a `https://doi.org/` prefix.
+      placeholder: ex. 10.5281/zenodo.15490547
+    validations:
+      required: false
   - type: input
     id: tags
     attributes:
       label: Tags
       description: Please provide relevant tags for the dataset, separated by commas.
       placeholder: ex. Orientation Mapping, Nanocrystals, Ptychography, Strain Mapping
     validations:
-      required: false
+      required: false
+  - type: dropdown
+    id: kind
+    attributes:
+      label: Kind
+      description: What the entry hands out - a dataset, or a trained model checkpoint.
+      default: 0
+      options:
+        - dataset
+        - weights
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: The released version this entry pins. One entry per version.
+      placeholder: ex. 1
+    validations:
+      required: false
+  - type: markdown
+    attributes:
+      value: |
+        The three fields below apply to `Kind: weights` only. Leave them blank for a
+        dataset - a dataset entry with a model block is rejected.
+  - type: input
+    id: model_class
+    attributes:
+      label: Model Class
+      description: The dotted import path of the class the checkpoint loads into. Required for model weights.
+      placeholder: ex. quantem.core.ml.CNN2d
+    validations:
+      required: false
+  - type: input
+    id: model_framework
+    attributes:
+      label: Model Framework
+      description: The framework the checkpoint was saved with. Required for model weights.
+      placeholder: ex. torch
+    validations:
+      required: false
+  - type: input
+    id: model_quantem
+    attributes:
+      label: Model quantem
+      description: The quantem versions the checkpoint loads under.
+      placeholder: ex. ">=0.2,<0.3"
+    validations:
+      required: false

.github/scripts/issue_to_yaml.py

@@ -1,130 +1,138 @@
 """Turn a filled-in "New Dataset" issue into a dataset YAML file.
 
-Run by ``.github/workflows/on_new_issue.yml``. The output is validated against
-``em_database/datasets/json-schema.json`` before it is written, so a malformed
-issue fails here rather than in the pull request the workflow opens.
+Run by ``.github/workflows/on_new_issue.yml``. The entry is assembled by
+``emdatabase.new_dataset.build_document``, so the issue route writes the same
+keys in the same order as the CLI and the docs form, and it goes through
+``emdatabase.metadata.validate_document`` before it is written - the same check
+the test suite and ``emdatabase.new_dataset`` run - so a malformed issue fails
+here rather than in the pull request the workflow opens.
 """
 
 import re
 import sys
-import urllib.request
 from pathlib import Path
 
-import jsonschema
 import yaml
 
-from em_database.metadata import check_vendor, load_schema, load_vendors
-
-FIELDS = {
-    "Dataset Name": r"--Dataset Name--\s*(.*)",
-    "Author": r"--Author--\s*(.*)",
-    "Affiliation": r"--Affiliation--\s*(.*)",
-    "URL": r"--URL--\s*(.*)",
-    "Checksum": r"--Checksum--\s*(.*)",
-    "Description": r"--Description--\s*([\s\S]*?)--Detector Manufacturer--",
-    "Detector Manufacturer": r"--Detector Manufacturer--\s*(.*)",
-    "Detector Model": r"Detector Model\s*(.*)",
-    "Microscope Vendor": r"Microscope Vendor\s*(.*)",
-    "Microscope Model": r"Microscope Model\s*(.*)",
-    "Accelerating Voltage": r"Accelerating Voltage\s*(.*)",
-    "Dataset License": r"Dataset License\s*(.*)",
-    "Technique": r"Technique\s*(.*)",
-    "Tags": r"Tags\s*(.*)",
-}
+from emdatabase.metadata import validate_document
+from emdatabase.new_dataset import build_document, content_length, split_url
+
+# The form's field labels, with the ``--...--`` markers stripped off.
+FIELDS = (
+    "Dataset Name",
+    "Author",
+    "Affiliation",
+    "ORCID",
+    "URL",
+    "File Name",
+    "Checksum",
+    "Description",
+    "Detector Manufacturer",
+    "Detector Model",
+    "Microscope Vendor",
+    "Microscope Model",
+    "Camera Length",
+    "Accelerating Voltage",
+    "Dataset License",
+    "Technique",
+    "DOI",
+    "Tags",
+    "Kind",
+    "Version",
+    "Model Class",
+    "Model Framework",
+    "Model quantem",
+)
 
 
 def clean(value):
     """Trim what GitHub adds around an issue-form answer.
 
-    Answers are separated by ``###`` headings, and the multi-line ones run up
-    to the next heading; an optional field left blank comes through as the
-    literal ``_No response_``.
+    An optional field left blank comes through as the literal ``_No response_``.
     """
-    value = value.strip().rstrip("#").strip()
+    value = value.strip()
     return "" if value == "_No response_" else value
 
 
 def parse_issue_body(text):
-    data = {}
-    for key, pattern in FIELDS.items():
-        match = re.search(pattern, text)
-        data[key] = clean(match.group(1)) if match else ""
-    return data
-
+    """The answers, keyed by field label, for everything the form asks.
 
-def content_length(url):
-    """The file's size in bytes, or None if the server will not say."""
-    request = urllib.request.Request(
-        url,
-        method="HEAD",
-        headers={"User-Agent": "em_database (https://github.com/CSSFrancis/em_data)"},
-    )
-    try:
-        with urllib.request.urlopen(request, timeout=60) as response:
-            length = response.headers.get("Content-Length")
-        return int(length) if length else None
-    except Exception:
-        return None
+    GitHub writes each answer under a ``###`` heading naming the field, so the
+    body is split on those headings rather than matched field by field. A
+    heading the form no longer has is ignored, and a field the body does not
+    carry comes back empty.
+    """
+    data = {name: "" for name in FIELDS}
+    for chunk in re.split(r"^#{1,6}[ \t]*", text, flags=re.MULTILINE)[1:]:
+        label, _, value = chunk.partition("\n")
+        key = label.strip().strip("-").strip()
+        if key in data:
+            data[key] = clean(value)
+    return data
 
 
 def build_yaml(data):
     """``(document, dataset_name)`` for a parsed issue."""
     name = re.sub(r"\W+", "", data["Dataset Name"])
     if not name:
         sys.exit("the issue has no dataset name")
-    # The form asks for a direct link to the file; the YAML wants the directory
-    # and the file name separately.
+    # The form asks for the download link; the YAML wants the directory and the
+    # file name separately, and keeps the whole link as `url` only when the file
+    # is not served at `source/file`.
     url = data["URL"].rstrip("/")
-    source, _, filename = url.rpartition("/")
-    if not source or not filename:
-        sys.exit(f"{data['URL']!r} is not a direct link to a file")
+    source, filename, link = split_url(url)
+    if not source:
+        sys.exit(f"{data['URL']!r} is not a link to a file")
+    filename = data["File Name"] or filename
+    if not filename:
+        sys.exit(f"{data['URL']!r} does not end in a file name; fill in --File Name--")
 
     entry = {
         "description": data["Description"],
         "source": source,
+        "url": link,
         "checksum": data["Checksum"],
         "file": filename,
         "size_bytes": content_length(url),
         "detector_manufacturer": data["Detector Manufacturer"],
         "detector": data["Detector Model"],
         "microscope_vendor": data["Microscope Vendor"],
         "microscope_model": data["Microscope Model"],
+        "camera_length": data["Camera Length"],
         "voltage": data["Accelerating Voltage"],
         "license": data["Dataset License"],
         "technique": data["Technique"],
+        "doi": data["DOI"],
         "tags": [t.strip() for t in data["Tags"].split(",") if t.strip()],
+        "version": data["Version"],
     }
     if data["Author"]:
-        entry["authors"] = {data["Author"]: {"affiliation": data["Affiliation"] or "Unspecified"}}
-    return {name: {k: v for k, v in entry.items() if v not in (None, "", [])}}, name
-
-
-def report_vendors(entry):
-    """Print vendor warnings; exit on a name that is a likely misspelling."""
-    vendors = load_vendors()
-    failed = False
-    for field, known in (
-        ("detector_manufacturer", vendors["detector_manufacturer"]),
-        ("microscope_vendor", vendors["microscope_vendor"]),
-    ):
-        result = check_vendor(entry.get(field, ""), known)
-        if result is None:
-            continue
-        level, message = result
-        print(f"{level}: {field}: {message}")
-        failed = failed or level == "error"
-    if failed:
-        sys.exit("fix the vendor spelling in the issue and reopen it")
+        author = {"affiliation": data["Affiliation"] or "Unspecified"}
+        if data["ORCID"]:
+            author["orcid"] = data["ORCID"]
+        entry["authors"] = {data["Author"]: author}
+    if data["Kind"] == "weights":
+        entry["kind"] = "weights"
+        model = {
+            "class": data["Model Class"],
+            "framework": data["Model Framework"],
+            "quantem": data["Model quantem"],
+        }
+        entry["model"] = {k: v for k, v in model.items() if v}
+    return build_document(name, entry), name
 
 
 if __name__ == "__main__":
     issue_file, out_dir = sys.argv[1], Path(sys.argv[2])
     document, dataset_name = build_yaml(parse_issue_body(Path(issue_file).read_text()))
-    jsonschema.validate(document, load_schema())
-    report_vendors(document[dataset_name])
-
     out_path = out_dir / f"{dataset_name}.yaml"
+    problems = validate_document(document, origin=out_path)
+    for problem in problems:
+        print(problem)
+    if problems:
+        sys.exit("fix the issue and reopen it")
+
     with open(out_path, "w") as f:
-        f.write("# $schema: ../json-schema.json\n")
+        f.write("# $schema: ./json-schema.json\n")
         yaml.dump(document, f, sort_keys=False)
     print(f"wrote {out_path}")

.github/workflows/build.yml

@@ -34,7 +34,7 @@ jobs:
         run: basedpyright
 
       - name: dataset stub is current
-        run: python -m em_database._create_stubs --check
+        run: python -m emdatabase._create_stubs --check
 
   build-with-pip:
     name: ${{ matrix.os }}-py${{ matrix.python-version }}${{ matrix.LABEL }}
@@ -84,7 +84,7 @@ jobs:
       - name: Run docstring tests
         continue-on-error: true
         run: |
-          pytest --doctest-modules --doctest-continue-on-failure --ignore-glob=em_database/tests em_database
+          pytest --doctest-modules --doctest-continue-on-failure --ignore-glob=emdatabase/tests emdatabase
 
       - name: Run tests
         run: |

.github/workflows/documentation.yaml

@@ -24,13 +24,13 @@ jobs:
       - name: Sphinx build
         timeout-minutes: 15
         run: |
-          make html -C doc
+          make html -C docs
 
       - name: Upload the built site
         uses: actions/upload-artifact@v4
         with:
-          name: html-${{ github.ref_name }}
-          path: doc/build/html
+          name: html-${{ github.run_id }}
+          path: docs/build/html
           retention-days: 14
 
       - name: Deploy to GitHub Pages
@@ -39,5 +39,5 @@ jobs:
         with:
           publish_branch: gh-pages
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir:  doc/build/html
+          publish_dir:  docs/build/html
           force_orphan: true