[WIP]: docs(huntsman): Document all supported configurations for the storage service.

docs/huntsman/conf/conf.py

@@ -1,5 +1,15 @@
 from datetime import date
 
+from docutils import nodes
+from docutils.parsers.rst import directives
+from sphinx.domains.std import ConfigurationValue
+
+# -- Repository information ----------------------------------------------------
+
+# NOTE: When a formal release is cut, update this to the corresponding release branch;
+# otherwise, it should stay on "main".
+SPIDER_GIT_REF = "main"
+
 # -- Project information -------------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
@@ -38,6 +48,10 @@
 # -- HTML output options -------------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
+# NOTE: Source copies are disabled since the theme renders no source links and the copies would
+# bypass the `source-read` substitutions below.
+html_copy_source = False
+
 html_favicon = "https://docs.yscope.com/_static/favicon.ico"
 html_title = project
 html_show_copyright = True
@@ -67,13 +81,54 @@
 html_context = {
     "github_user": "y-scope",
     "github_repo": "spider",
-    "github_version": "main",
+    "github_version": SPIDER_GIT_REF,
     "doc_path": "docs/huntsman/src",
 }
 
+# -- Custom directives ---------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/extdev/domainapi.html
+
+_CONFVAL_EXTRA_OPTIONS = ("Helm value", "Docker Compose env var")
+
+
+class _SpiderConfigurationValue(ConfigurationValue):
+    option_spec = {
+        **ConfigurationValue.option_spec,
+        **{name: directives.unchanged_required for name in _CONFVAL_EXTRA_OPTIONS},
+    }
+
+    def transform_content(self, content_node):
+        super().transform_content(content_node)
+        fields = []
+        for name in _CONFVAL_EXTRA_OPTIONS:
+            if name not in self.options:
+                continue
+            parsed, msgs = self.parse_inline(self.options[name], lineno=self.lineno)
+            fields.append(
+                nodes.field("", nodes.field_name("", name), nodes.field_body("", *parsed))
+            )
+            fields.extend(msgs)
+        if not fields:
+            return
+        if content_node.children and isinstance(content_node.children[0], nodes.field_list):
+            content_node.children[0][:0] = fields
+        else:
+            content_node.insert(0, nodes.field_list("", *fields))
+
+
+# -- Source transforms ---------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/extdev/event_callbacks.html#event-source-read
+
+
+def _substitute_docs_vars(app, docname, source):
+    source[0] = source[0].replace("DOCS_VAR_SPIDER_GIT_REF", SPIDER_GIT_REF)
+
+
 # -- Theme custom CSS and JS ---------------------------------------------------
 # https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/static_assets.html
 
 
 def setup(app):
     app.add_css_file("custom.css")
+    app.add_directive_to_domain("std", "confval", _SpiderConfigurationValue, override=True)
+    app.connect("source-read", _substitute_docs_vars)

docs/huntsman/src/user-docs/guides-configuration/index.md

@@ -6,6 +6,10 @@
 
 The guides below describe how to configure each of Spider's components.
 
+Spider's components expose a range of configuration values that let you tune the system's behavior,
+such as its expected memory footprint or its scheduling latency. The guides below document the
+values that each deployment exposes, along with the default that the deployment ships.
+
 ::::{grid} 1 1 2 2
 :gutter: 2
 
@@ -38,3 +42,66 @@ storage.md
 scheduler.md
 worker.md
 :::
+
+The deployments ship a default for every value they expose, so you only need to override the ones
+you want to change. Some settings are managed by the deployment itself and can't be overridden;
+those are marked `Not supported` in the guides. How you apply an override depends on your deployment
+type.
+
+## Kubernetes deployment
+
+The Helm chart's defaults live in [`values.yaml`][helm-values]. There are two ways to override them:
+
+* **Option 1: A custom values file**—Create a file containing only the keys you want to change
+  and pass it with `-f`. Helm deep-merges it into the chart's `values.yaml`, so you don't need to
+  copy the whole file. `-f` can be repeated, and later files take precedence.
+
+  ```shell
+  helm upgrade --install spider tools/deployment/spider-helm -f my-values.yaml
+  ```
+
+* **Option 2: Override specific values**—Use Helm's [`--set`][helm-set] option to override
+  individual keys when installing or upgrading the chart.
+
+  ```shell
+  helm upgrade --install spider tools/deployment/spider-helm \
+      --set spiderConfig.storage.runtime.inbound_queue.task_capacity=1048576
+  ```
+
+A few caveats:
+
+* List-valued settings, such as `spiderConfig.bundled`, are replaced wholesale rather than merged.
+* `--set` infers the type of each value; use `--set-string` for values that must stay strings, such
+  as image tags.
+* On `helm upgrade`, once you pass any `-f` or `--set`, the values you don't re-supply fall back to
+  the chart's defaults. Re-pass the same values file every time, or use `--reuse-values`.
+* An upgrade that only changes a config value updates the chart's ConfigMap but doesn't restart the
+  pods, so the new config never reaches a running container. Roll the affected components yourself,
+  e.g. `kubectl rollout restart deployment/spider-storage`. Run `kubectl get deployments` to find
+  the exact names, which depend on the release name.
+
+For each value in the guides, the `Helm value` field names the key to set.
+
+## Docker Compose deployment
+
+Docker Compose renders each component's config file from a template, interpolating environment
+variables into it. A few settings, such as the database credentials and the log level, are passed to
+the containers as environment variables instead.
+
+[`.env.example`][env-example] lists all supported environment variables and their default values. To
+override a default, copy the file to `.env` in the same directory as `compose.yaml`, edit the values
+you care about, and recreate the containers:
+
+```shell
+cp .env.example .env
+docker compose up -d
+```
+
+Creating `.env` is optional: every compose file embeds `${VAR:-default}`, so the stack runs on the
+defaults if no `.env` exists. Exported shell variables and `--env-file` work as well.
+
+For each value in the guides, the `Docker Compose env var` field names the variable to set.
+
+[env-example]: https://github.com/y-scope/spider/blob/DOCS_VAR_SPIDER_GIT_REF/tools/deployment/spider-compose/.env.example
+[helm-set]: https://helm.sh/docs/helm/helm_install/
+[helm-values]: https://github.com/y-scope/spider/blob/DOCS_VAR_SPIDER_GIT_REF/tools/deployment/spider-helm/values.yaml