fix: warn when documented columns are missing from the relation in persist_docs

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Avoid treating unchanged Streaming Table `databricks_tags` as configuration changes by diffing against existing table tags. ([#1602](https://github.com/databricks/dbt-databricks/pull/1602) resolves [#1601](https://github.com/databricks/dbt-databricks/issues/1601))
 - Allow dropping a column that has governed tags ([#1597](https://github.com/databricks/dbt-databricks/pull/1597) resolves [#1323](https://github.com/databricks/dbt-databricks/issues/1323))
 - Fix view materialization incorrectly producing a no-op instead of forcing recreation when `--full-refresh` is provided alongside `view_update_via_alter: true` and `use_materialization_v2: true` ([#1456](https://github.com/databricks/dbt-databricks/pull/1456) resolves [#1404](https://github.com/databricks/dbt-databricks/issues/1404))
+- Warn when a column documented in a model's `schema.yml` is absent from the relation, instead of silently skipping it — surfaces typos and stale column documentation. The check runs post-build against the actual relation (matching the shared `validate_doc_columns` behavior the other adapters use), so it covers V1 and V2 table/incremental on both the initial create and subsequent runs, does not false-warn on a legitimately new column, and is gated on `persist_docs.columns`. Also fixes the V2 column-comment gate to key off `persist_docs.columns` rather than `persist_docs.relation`. Materialized-view/streaming-table and view create are tracked as follow-ups. Ports the behavior added in dbt-adapters ([dbt-adapters#1684](https://github.com/dbt-labs/dbt-adapters/pull/1684) closes [dbt-adapters#1690](https://github.com/dbt-labs/dbt-adapters/issues/1690)) ([#1563](https://github.com/databricks/dbt-databricks/pull/1563)).
 - Support `dbt clone` and rebuilds over an existing shallow clone ([#1592](https://github.com/databricks/dbt-databricks/pull/1592) resolves [#1165](https://github.com/databricks/dbt-databricks/issues/1165), follow-up [#1608](https://github.com/databricks/dbt-databricks/pull/1608))
 - Fix managed Iceberg Python models failing with `MANAGED_TABLE_FORMAT` by emitting `.format("iceberg")` instead of the `parquet` sentinel from `resolve_file_format` (thanks @Divya-Kovvuru-0802!) ([#1593](https://github.com/databricks/dbt-databricks/pull/1593) resolves [#1591](https://github.com/databricks/dbt-databricks/issues/1591))
 - Quote generated column identifiers in incremental strategies so non-ASCII column names no longer fail on subsequent runs ([#1595](https://github.com/databricks/dbt-databricks/pull/1595) resolves [#1594](https://github.com/databricks/dbt-databricks/issues/1594))

dbt/adapters/databricks/impl.py

@@ -3,7 +3,7 @@
 import re
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from collections.abc import Iterable, Iterator
+from collections.abc import Iterable, Iterator, Mapping
 from concurrent.futures import Future
 from contextlib import contextmanager
 from dataclasses import dataclass, field
@@ -57,6 +57,10 @@
 from dbt.adapters.databricks.global_state import GlobalState
 from dbt.adapters.databricks.handle import SqlUtils
 from dbt.adapters.databricks.logging import logger
+from dbt.adapters.databricks.persist_doc_column_warnings import (
+    reset_missing_persist_doc_column_warnings,
+    warn_missing_persist_doc_columns,
+)
 from dbt.adapters.databricks.python_models.python_submissions import (
     AllPurposeClusterPythonJobHelper,
     JobClusterPythonJobHelper,
@@ -902,6 +906,11 @@ def get_behavior_flag_no_warn(self, behavior_flag_name: str) -> bool:
         behavior_flag = getattr(self.behavior, behavior_flag_name)
         return behavior_flag.no_warn
 
+    def pre_model_hook(self, config: Mapping[str, Any]) -> Any:
+        """Reset missing-column warn dedupe so each model materialization can warn once."""
+        reset_missing_persist_doc_column_warnings()
+        return super().pre_model_hook(config)
+
     @available.parse(lambda *a, **k: (None, None))
     @record_function(
         DatabricksAdapterAddQueryRecord,
@@ -994,6 +1003,26 @@ def _catalog(self, catalog: Optional[str]) -> Iterator[None]:
             if current_catalog is not None:
                 self.execute_macro(USE_CATALOG_MACRO_NAME, kwargs=dict(catalog=current_catalog))
 
+    @staticmethod
+    def _find_missing_doc_columns(
+        existing_columns: list[DatabricksColumn], columns: dict[str, Any]
+    ) -> list[str]:
+        """Documented column names (from the model) that are absent from the relation."""
+        existing_lower = {column.column.lower() for column in existing_columns}
+        return [name for name in columns if name.lower() not in existing_lower]
+
+    @available.parse(lambda *a, **k: None)
+    def validate_persist_doc_columns(
+        self, existing_columns: list[DatabricksColumn], columns: dict[str, Any]
+    ) -> None:
+        """Warn about documented columns that are absent from the relation.
+
+        Mirrors the shared ``validate_doc_columns`` behavior the other adapters use: the check runs
+        post-build against the actual relation, so a legitimately new column (present in the model
+        and the freshly-built relation) is not flagged. Applies no comments.
+        """
+        warn_missing_persist_doc_columns(self._find_missing_doc_columns(existing_columns, columns))
+
     @available.parse(lambda *a, **k: {})
     def get_persist_doc_columns(
         self, existing_columns: list[DatabricksColumn], columns: dict[str, Any]
@@ -1008,6 +1037,12 @@ def get_persist_doc_columns(
         # Create a case-insensitive lookup for column names
         columns_lower = {k.lower(): k for k in columns.keys()}
 
+        # Documented-but-absent columns are skipped below (rather than erroring on the alter); warn
+        # so the user can catch typos and stale documentation. This runs post-build (V1 persist_docs
+        # gathers existing_columns from the written relation), so a legitimately new column is not
+        # flagged.
+        warn_missing_persist_doc_columns(self._find_missing_doc_columns(existing_columns, columns))
+
         for column in existing_columns:
             name = column.column
             # Use case-insensitive comparison for column names

dbt/adapters/databricks/persist_doc_column_warnings.py

@@ -0,0 +1,54 @@
+"""Shared warning for documented columns absent from the relation.
+
+Both the V1 persist_docs helper (``get_persist_doc_columns``) and the V2 changeset
+helper (``ColumnCommentsConfig.get_diff``) need to surface the same user-facing
+warning. On V1 incremental subsequent runs those two paths can both execute in a
+single model materialization; dedupe so the message appears exactly once per
+unique missing set **within that materialization**.
+
+State is thread-local: dbt assigns each model to one worker thread, so parallel
+runs do not clear or suppress each other's warnings. ``pre_model_hook`` resets
+the cache at the start of each model on that thread.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Sequence
+
+from dbt.adapters.events.types import AdapterEventWarning
+from dbt_common.events.functions import warn_or_error
+
+_thread_state = threading.local()
+
+
+def _emitted_keys() -> set[str]:
+    keys = getattr(_thread_state, "emitted_missing_keys", None)
+    if keys is None:
+        keys = set()
+        _thread_state.emitted_missing_keys = keys
+    return keys
+
+
+def reset_missing_persist_doc_column_warnings() -> None:
+    """Clear the per-materialization dedupe cache for the current thread."""
+    _thread_state.emitted_missing_keys = set()
+
+
+def warn_missing_persist_doc_columns(missing: Sequence[str]) -> None:
+    """Warn once per unique set of documented-but-absent column names (per thread)."""
+    if not missing:
+        return
+    key = ", ".join(sorted(missing, key=str.lower))
+    emitted = _emitted_keys()
+    if key in emitted:
+        return
+    emitted.add(key)
+    warn_or_error(
+        AdapterEventWarning(
+            base_msg=(
+                "The following columns are specified in the schema but are not present "
+                "in the database and will be skipped: " + ", ".join(missing)
+            )
+        )
+    )

dbt/adapters/databricks/relation_configs/column_comments.py

@@ -24,7 +24,13 @@ def get_diff(self, other: "ColumnCommentsConfig") -> Optional["ColumnCommentsCon
             other_comments_lower = {k.lower(): v for k, v in other.comments.items()}
 
             for column_name, comment in self.comments.items():
-                # Use case-insensitive comparison for column names
+                # Use case-insensitive comparison for column names. Documented columns that are
+                # absent from the relation are skipped here so the alter never targets a nonexistent
+                # column; the user-facing "missing column" warning is emitted post-build by
+                # validate_persist_doc_columns (against the actual relation), so a legitimately new
+                # column is not flagged before it has been materialized.
+                if column_name.lower() not in other_comments_lower:
+                    continue
                 other_comment = other_comments_lower.get(column_name.lower())
                 if comment != other_comment:
                     column_name = f"`{column_name}`"
@@ -53,7 +59,10 @@ def from_relation_config(cls, relation_config: RelationConfig) -> ColumnComments
         columns = getattr(relation_config, "columns", {})
         persist = False
         if relation_config.config:
-            persist = relation_config.config.persist_docs.get("relation") or False
+            # Column comments are gated on persist_docs.columns (the column-level knob), matching
+            # config.persist_column_docs() used by the V1 persist_docs / view-create / seed paths.
+            # persist_docs.relation is the table-comment knob and is the wrong gate here.
+            persist = relation_config.config.persist_docs.get("columns") or False
         comments = {}
         for column_name, column in columns.items():
             if hasattr(column, "description"):

dbt/include/databricks/macros/adapters/persist_docs.sql

@@ -42,6 +42,23 @@
   {% endif %}
 {% endmacro %}
 
+{#--
+  Post-build validation of documented column comments against the actual relation.
+
+  The V2 materialization path applies column comments inline at create-time and via the
+  relation-config diff (neither of which sees the model's documented columns as a set), so this
+  runs after the relation is built to surface columns that are documented in the schema but absent
+  from the relation (typos / stale docs). It mirrors the shared validate_doc_columns behavior the
+  other adapters use, and applies no comments itself. Gated on persist_docs.columns so it never
+  fires when column persistence is disabled (avoids --warn-error false failures).
+--#}
+{% macro validate_persist_doc_columns(relation, model) -%}
+  {% if config.persist_column_docs() and model.columns %}
+    {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}
+    {%- do adapter.validate_persist_doc_columns(existing_columns, model.columns) -%}
+  {% endif %}
+{%- endmacro %}
+
 {% macro alter_relation_comment_sql(relation, description) %}
 COMMENT ON {{ relation.type.render().upper() }} {{ relation.render() }} IS '{{ description | replace("'", "\\'") }}'
 {% endmacro %}

dbt/include/databricks/macros/materializations/incremental/incremental.sql

@@ -81,6 +81,10 @@
       {%- endif -%}
     {%- endif -%}
 
+    {#-- Placed here so it runs on every sub-branch above (create/replace/merge) and regardless of
+         incremental_apply_config_changes. --#}
+    {% do validate_persist_doc_columns(target_relation, model) %}
+
     {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
     {% do apply_grants(target_relation, grant_config, should_revoke) %}
     {% do optimize(target_relation) %}

dbt/include/databricks/macros/materializations/table.sql

@@ -32,6 +32,8 @@
       {% endif %}
     {% endif %}
 
+    {% do validate_persist_doc_columns(target_relation, model) %}
+
     {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
     {{ apply_grants(target_relation, grant_config, should_revoke) }}
     {% do optimize(target_relation) %}

tests/functional/adapter/persist_docs/fixtures.py

@@ -39,6 +39,25 @@
 select 1 as id, 'alice' as name
 """
 
+# Incremental model whose schema documents a column absent from the relation. Used to exercise the
+# V2 alter/changeset path (ColumnCommentsConfig.get_diff), which — unlike a table rebuild — is only
+# reached on a subsequent run against an existing relation.
+missing_column_incremental_sql = """
+{{ config(materialized='incremental') }}
+select 1 as id, 'Ed' as name
+"""
+
+missing_column_incremental_schema = """
+version: 2
+models:
+  - name: missing_column_incremental
+    columns:
+      - name: id
+        description: "test id column description"
+      - name: column_that_does_not_exist
+        description: "comment that cannot be created"
+"""
+
 gate_model_schema = """
 version: 2
 models: