From e6926f6b0b59232d6824e2929dc3bcdb4540cc48 Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 4 May 2026 16:47:14 +0530 Subject: [PATCH 01/10] feat: declare CATALOG_V2_CONFIGS for catalogs.yml v2 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds UnityDatabricksConfig and HiveMetastoreDatabricksConfig dataclasses and declares them on DatabricksAdapter.CATALOG_V2_CONFIGS. This enables parse-time validation of catalogs.yml v2 when the use_catalogs_v2 behavior flag is set in dbt-core. UnityDatabricksConfig enforces file_format='parquet' when use_uniform is false/unset, and 'delta' when use_uniform=true. HiveMetastoreDatabricksConfig accepts delta, parquet, or hudi. Both schemas are dbtClassMixin dataclasses — structural validation (unknown keys, required fields) via jsonschema; semantic constraints in __post_init__. --- CHANGELOG.md | 1 + dbt/adapters/databricks/catalogs/__init__.py | 6 ++ dbt/adapters/databricks/catalogs/_v2.py | 38 ++++++++ dbt/adapters/databricks/impl.py | 6 ++ tests/unit/test_v2_catalog_configs.py | 93 ++++++++++++++++++++ 5 files changed, 144 insertions(+) create mode 100644 dbt/adapters/databricks/catalogs/_v2.py create mode 100644 tests/unit/test_v2_catalog_configs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 87659b60a..41d15d69e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +- Declare `CATALOG_V2_CONFIGS` on `DatabricksAdapter` with `UnityDatabricksConfig` and `HiveMetastoreDatabricksConfig` to support parse-time validation of catalogs.yml v2 (requires `use_catalogs_v2` behavior flag in dbt-core) - Add `invocation_id` to the default query comment ([#1377](https://github.com/databricks/dbt-databricks/issues/1377)) ### Fixes diff --git a/dbt/adapters/databricks/catalogs/__init__.py b/dbt/adapters/databricks/catalogs/__init__.py index a6f1bed39..e06a6877c 100644 --- a/dbt/adapters/databricks/catalogs/__init__.py +++ b/dbt/adapters/databricks/catalogs/__init__.py @@ -1,9 +1,15 @@ from dbt.adapters.databricks.catalogs._hive_metastore import HiveMetastoreCatalogIntegration from dbt.adapters.databricks.catalogs._relation import DatabricksCatalogRelation from dbt.adapters.databricks.catalogs._unity import UnityCatalogIntegration +from dbt.adapters.databricks.catalogs._v2 import ( + HiveMetastoreDatabricksConfig, + UnityDatabricksConfig, +) __all__ = [ "DatabricksCatalogRelation", "HiveMetastoreCatalogIntegration", + "HiveMetastoreDatabricksConfig", "UnityCatalogIntegration", + "UnityDatabricksConfig", ] diff --git a/dbt/adapters/databricks/catalogs/_v2.py b/dbt/adapters/databricks/catalogs/_v2.py new file mode 100644 index 000000000..74f722f4f --- /dev/null +++ b/dbt/adapters/databricks/catalogs/_v2.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass +from typing import Optional + +from dbt_common.dataclass_schema import dbtClassMixin +from dbt_common.exceptions import DbtValidationError + + +@dataclass +class UnityDatabricksConfig(dbtClassMixin): + file_format: str + location_root: Optional[str] = None + use_uniform: Optional[bool] = None + + def __post_init__(self) -> None: + if not self.file_format.strip(): + raise DbtValidationError("'file_format' must be non-empty") + # file_format depends on use_uniform — see dbt-adapters issue #9648 + if self.use_uniform: + if self.file_format.lower() != "delta": + raise DbtValidationError("file_format must be 'delta' when 'use_uniform' is true") + else: + if self.file_format.lower() != "parquet": + raise DbtValidationError( + "file_format must be 'parquet' when 'use_uniform' is false or unset" + ) + if self.location_root is not None and not self.location_root.strip(): + raise DbtValidationError("'location_root' cannot be blank") + + +@dataclass +class HiveMetastoreDatabricksConfig(dbtClassMixin): + file_format: str + + def __post_init__(self) -> None: + if self.file_format.lower() not in {"delta", "parquet", "hudi"}: + raise DbtValidationError( + f"file_format must be one of: {sorted(f.upper() for f in {'delta', 'parquet', 'hudi'})}" + ) \ No newline at end of file diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index c9f39a203..ced7159ac 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -41,6 +41,8 @@ GetColumnsByDescribe, ) from dbt.adapters.databricks.catalogs import ( + HiveMetastoreDatabricksConfig, + UnityDatabricksConfig, DatabricksCatalogRelation, HiveMetastoreCatalogIntegration, UnityCatalogIntegration, @@ -233,6 +235,10 @@ class DatabricksAdapter(SparkAdapter): HiveMetastoreCatalogIntegration, UnityCatalogIntegration, ] + CATALOG_V2_CONFIGS = { + "unity": UnityDatabricksConfig, + "hive_metastore": HiveMetastoreDatabricksConfig, + } CONSTRAINT_SUPPORT = constraints.CONSTRAINT_SUPPORT get_column_behavior: GetColumnsBehavior diff --git a/tests/unit/test_v2_catalog_configs.py b/tests/unit/test_v2_catalog_configs.py new file mode 100644 index 000000000..c5f8d7d9a --- /dev/null +++ b/tests/unit/test_v2_catalog_configs.py @@ -0,0 +1,93 @@ +import pytest + +from dbt.adapters.databricks.catalogs._v2 import ( + HiveMetastoreDatabricksConfig, + UnityDatabricksConfig, +) +from dbt.adapters.databricks.impl import DatabricksAdapter +from dbt_common.exceptions import DbtValidationError + + +# ===== CATALOG_V2_CONFIGS class attribute ===== + + +def test_unity_registered(): + assert DatabricksAdapter.CATALOG_V2_CONFIGS["unity"] is UnityDatabricksConfig + + +def test_hive_metastore_registered(): + assert ( + DatabricksAdapter.CATALOG_V2_CONFIGS["hive_metastore"] is HiveMetastoreDatabricksConfig + ) + + +# ===== UnityDatabricksConfig ===== + + +def test_unity_parquet_without_uniform(): + cfg = UnityDatabricksConfig(file_format="parquet") + assert cfg.file_format == "parquet" + + +def test_unity_delta_with_uniform(): + cfg = UnityDatabricksConfig(file_format="delta", use_uniform=True) + assert cfg.file_format == "delta" + assert cfg.use_uniform is True + + +def test_unity_with_location_root(): + cfg = UnityDatabricksConfig(file_format="parquet", location_root="/mnt/data") + assert cfg.location_root == "/mnt/data" + + +def test_unity_delta_without_uniform_raises(): + with pytest.raises(DbtValidationError, match="file_format must be 'parquet'"): + UnityDatabricksConfig(file_format="delta") + + +def test_unity_parquet_with_uniform_raises(): + with pytest.raises(DbtValidationError, match="file_format must be 'delta'"): + UnityDatabricksConfig(file_format="parquet", use_uniform=True) + + +def test_unity_blank_file_format_raises(): + with pytest.raises(DbtValidationError, match="file_format.*non-empty"): + UnityDatabricksConfig(file_format=" ") + + +def test_unity_blank_location_root_raises(): + with pytest.raises(DbtValidationError, match="location_root.*blank"): + UnityDatabricksConfig(file_format="parquet", location_root=" ") + + +def test_unity_rejects_unknown_keys(): + with pytest.raises(Exception, match="Additional properties"): + UnityDatabricksConfig.validate({"file_format": "parquet", "bogus": True}) + + +# ===== HiveMetastoreDatabricksConfig ===== + + +def test_hive_delta_valid(): + cfg = HiveMetastoreDatabricksConfig(file_format="delta") + assert cfg.file_format == "delta" + + +def test_hive_parquet_valid(): + cfg = HiveMetastoreDatabricksConfig(file_format="parquet") + assert cfg.file_format == "parquet" + + +def test_hive_hudi_valid(): + cfg = HiveMetastoreDatabricksConfig(file_format="hudi") + assert cfg.file_format == "hudi" + + +def test_hive_invalid_file_format_raises(): + with pytest.raises(DbtValidationError, match="file_format must be one of"): + HiveMetastoreDatabricksConfig(file_format="avro") + + +def test_hive_rejects_unknown_keys(): + with pytest.raises(Exception, match="Additional properties"): + HiveMetastoreDatabricksConfig.validate({"file_format": "delta", "extra": "bad"}) \ No newline at end of file From e2d06887e0a93c404279162f6b0da9a5e0ad74c3 Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Tue, 5 May 2026 16:30:02 +0530 Subject: [PATCH 02/10] Remove stale comment from UnityDatabricksConfig --- dbt/adapters/databricks/catalogs/_v2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/databricks/catalogs/_v2.py b/dbt/adapters/databricks/catalogs/_v2.py index 74f722f4f..e225d8d75 100644 --- a/dbt/adapters/databricks/catalogs/_v2.py +++ b/dbt/adapters/databricks/catalogs/_v2.py @@ -14,7 +14,6 @@ class UnityDatabricksConfig(dbtClassMixin): def __post_init__(self) -> None: if not self.file_format.strip(): raise DbtValidationError("'file_format' must be non-empty") - # file_format depends on use_uniform — see dbt-adapters issue #9648 if self.use_uniform: if self.file_format.lower() != "delta": raise DbtValidationError("file_format must be 'delta' when 'use_uniform' is true") From 0e119f60ef4333af34f2761b2d8f6b58fef299a8 Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Tue, 5 May 2026 16:59:23 +0530 Subject: [PATCH 03/10] minor fixes --- dbt/adapters/databricks/catalogs/_v2.py | 4 ++-- dbt/adapters/databricks/impl.py | 4 ++-- tests/unit/test_v2_catalog_configs.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/databricks/catalogs/_v2.py b/dbt/adapters/databricks/catalogs/_v2.py index e225d8d75..a2456ef72 100644 --- a/dbt/adapters/databricks/catalogs/_v2.py +++ b/dbt/adapters/databricks/catalogs/_v2.py @@ -33,5 +33,5 @@ class HiveMetastoreDatabricksConfig(dbtClassMixin): def __post_init__(self) -> None: if self.file_format.lower() not in {"delta", "parquet", "hudi"}: raise DbtValidationError( - f"file_format must be one of: {sorted(f.upper() for f in {'delta', 'parquet', 'hudi'})}" - ) \ No newline at end of file + f"file_format must be one of: {sorted({'delta', 'parquet', 'hudi'})}" + ) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index ced7159ac..6d78ca46a 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -41,11 +41,11 @@ GetColumnsByDescribe, ) from dbt.adapters.databricks.catalogs import ( - HiveMetastoreDatabricksConfig, - UnityDatabricksConfig, DatabricksCatalogRelation, HiveMetastoreCatalogIntegration, + HiveMetastoreDatabricksConfig, UnityCatalogIntegration, + UnityDatabricksConfig, ) from dbt.adapters.databricks.column import DatabricksColumn from dbt.adapters.databricks.connections import ( diff --git a/tests/unit/test_v2_catalog_configs.py b/tests/unit/test_v2_catalog_configs.py index c5f8d7d9a..814edf85d 100644 --- a/tests/unit/test_v2_catalog_configs.py +++ b/tests/unit/test_v2_catalog_configs.py @@ -90,4 +90,4 @@ def test_hive_invalid_file_format_raises(): def test_hive_rejects_unknown_keys(): with pytest.raises(Exception, match="Additional properties"): - HiveMetastoreDatabricksConfig.validate({"file_format": "delta", "extra": "bad"}) \ No newline at end of file + HiveMetastoreDatabricksConfig.validate({"file_format": "delta", "extra": "bad"}) From adfc0ee8bdb449112854cd230900314eae9dee5a Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 11 May 2026 10:48:11 +0530 Subject: [PATCH 04/10] Rearchitect to adapter-owned bridge: replace CATALOG_V2_CONFIGS with hook methods - Remove CATALOG_V2_CONFIGS and _v2.py typed dataclasses (old approach) - Add Capability.CatalogsV2 and _v2_to_v1_type hook (new approach) - Move use_uniform x file_format validation into UnityCatalogIntegration.__init__ - Move file_format enum validation into HiveMetastoreCatalogIntegration.__init__ --- dbt/adapters/databricks/catalogs/__init__.py | 8 +--- .../databricks/catalogs/_hive_metastore.py | 8 ++++ dbt/adapters/databricks/catalogs/_unity.py | 15 ++++++++ dbt/adapters/databricks/catalogs/_v2.py | 37 ------------------- dbt/adapters/databricks/impl.py | 12 +++--- 5 files changed, 31 insertions(+), 49 deletions(-) delete mode 100644 dbt/adapters/databricks/catalogs/_v2.py diff --git a/dbt/adapters/databricks/catalogs/__init__.py b/dbt/adapters/databricks/catalogs/__init__.py index e06a6877c..c434823a6 100644 --- a/dbt/adapters/databricks/catalogs/__init__.py +++ b/dbt/adapters/databricks/catalogs/__init__.py @@ -1,15 +1,9 @@ from dbt.adapters.databricks.catalogs._hive_metastore import HiveMetastoreCatalogIntegration from dbt.adapters.databricks.catalogs._relation import DatabricksCatalogRelation from dbt.adapters.databricks.catalogs._unity import UnityCatalogIntegration -from dbt.adapters.databricks.catalogs._v2 import ( - HiveMetastoreDatabricksConfig, - UnityDatabricksConfig, -) __all__ = [ "DatabricksCatalogRelation", "HiveMetastoreCatalogIntegration", - "HiveMetastoreDatabricksConfig", "UnityCatalogIntegration", - "UnityDatabricksConfig", -] +] \ No newline at end of file diff --git a/dbt/adapters/databricks/catalogs/_hive_metastore.py b/dbt/adapters/databricks/catalogs/_hive_metastore.py index d5ee31356..caace7c2d 100644 --- a/dbt/adapters/databricks/catalogs/_hive_metastore.py +++ b/dbt/adapters/databricks/catalogs/_hive_metastore.py @@ -2,10 +2,13 @@ from dbt.adapters.catalogs import CatalogIntegration, CatalogIntegrationConfig from dbt.adapters.contracts.relation import RelationConfig +from dbt_common.exceptions import DbtValidationError from dbt.adapters.databricks import constants, parse_model from dbt.adapters.databricks.catalogs._relation import DatabricksCatalogRelation +_VALID_HIVE_FILE_FORMATS = {"delta", "parquet", "hudi"} + class HiveMetastoreCatalogIntegration(CatalogIntegration): catalog_type = constants.HIVE_METASTORE_CATALOG_TYPE @@ -14,6 +17,11 @@ class HiveMetastoreCatalogIntegration(CatalogIntegration): def __init__(self, config: CatalogIntegrationConfig) -> None: super().__init__(config) self.file_format: Optional[str] = config.file_format + if config.file_format and config.file_format.lower() not in _VALID_HIVE_FILE_FORMATS: + raise DbtValidationError( + f"Catalog '{config.name}' hive_metastore/databricks file_format " + f"must be one of {sorted(_VALID_HIVE_FILE_FORMATS)}, got '{config.file_format}'" + ) @property def location_root(self) -> Optional[str]: diff --git a/dbt/adapters/databricks/catalogs/_unity.py b/dbt/adapters/databricks/catalogs/_unity.py index 9e650bea8..b68b68c12 100644 --- a/dbt/adapters/databricks/catalogs/_unity.py +++ b/dbt/adapters/databricks/catalogs/_unity.py @@ -2,6 +2,7 @@ from dbt.adapters.catalogs import CatalogIntegration, CatalogIntegrationConfig from dbt.adapters.contracts.relation import RelationConfig +from dbt_common.exceptions import DbtValidationError from dbt.adapters.databricks import constants, parse_model from dbt.adapters.databricks.catalogs._relation import DatabricksCatalogRelation @@ -14,8 +15,22 @@ class UnityCatalogIntegration(CatalogIntegration): def __init__(self, config: CatalogIntegrationConfig) -> None: super().__init__(config) if location_root := config.adapter_properties.get("location_root"): + if not str(location_root).strip(): + raise DbtValidationError( + f"Catalog '{config.name}' unity/databricks location_root cannot be blank" + ) self.external_volume: Optional[str] = location_root self.file_format: Optional[str] = config.file_format + use_uniform = config.adapter_properties.get("use_uniform", False) + ff = (config.file_format or "").lower() + if use_uniform and ff != "delta": + raise DbtValidationError( + f"Catalog '{config.name}' unity/databricks use_uniform: true requires file_format: delta" + ) + if not use_uniform and ff and ff != "parquet": + raise DbtValidationError( + f"Catalog '{config.name}' unity/databricks use_uniform: false (or unset) requires file_format: parquet" + ) @property def location_root(self) -> Optional[str]: diff --git a/dbt/adapters/databricks/catalogs/_v2.py b/dbt/adapters/databricks/catalogs/_v2.py deleted file mode 100644 index a2456ef72..000000000 --- a/dbt/adapters/databricks/catalogs/_v2.py +++ /dev/null @@ -1,37 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - -from dbt_common.dataclass_schema import dbtClassMixin -from dbt_common.exceptions import DbtValidationError - - -@dataclass -class UnityDatabricksConfig(dbtClassMixin): - file_format: str - location_root: Optional[str] = None - use_uniform: Optional[bool] = None - - def __post_init__(self) -> None: - if not self.file_format.strip(): - raise DbtValidationError("'file_format' must be non-empty") - if self.use_uniform: - if self.file_format.lower() != "delta": - raise DbtValidationError("file_format must be 'delta' when 'use_uniform' is true") - else: - if self.file_format.lower() != "parquet": - raise DbtValidationError( - "file_format must be 'parquet' when 'use_uniform' is false or unset" - ) - if self.location_root is not None and not self.location_root.strip(): - raise DbtValidationError("'location_root' cannot be blank") - - -@dataclass -class HiveMetastoreDatabricksConfig(dbtClassMixin): - file_format: str - - def __post_init__(self) -> None: - if self.file_format.lower() not in {"delta", "parquet", "hudi"}: - raise DbtValidationError( - f"file_format must be one of: {sorted({'delta', 'parquet', 'hudi'})}" - ) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 345885c19..f8db98439 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -43,9 +43,7 @@ from dbt.adapters.databricks.catalogs import ( DatabricksCatalogRelation, HiveMetastoreCatalogIntegration, - HiveMetastoreDatabricksConfig, UnityCatalogIntegration, - UnityDatabricksConfig, ) from dbt.adapters.databricks.column import DatabricksColumn from dbt.adapters.databricks.connections import ( @@ -228,6 +226,7 @@ class DatabricksAdapter(SparkAdapter): { Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), + Capability.CatalogsV2: CapabilitySupport(support=Support.Full), } ) @@ -235,9 +234,9 @@ class DatabricksAdapter(SparkAdapter): HiveMetastoreCatalogIntegration, UnityCatalogIntegration, ] - CATALOG_V2_CONFIGS = { - "unity": UnityDatabricksConfig, - "hive_metastore": HiveMetastoreDatabricksConfig, + _V2_TO_V1_TYPE: ClassVar[dict[str, str]] = { + "unity": constants.UNITY_CATALOG_TYPE, + "hive_metastore": constants.HIVE_METASTORE_CATALOG_TYPE, } CONSTRAINT_SUPPORT = constraints.CONSTRAINT_SUPPORT @@ -258,6 +257,9 @@ def __init__(self, config: Any, mp_context: SpawnContext) -> None: self.get_behavior_flag_no_warn(USE_MANAGED_ICEBERG["name"]) ) + def _v2_to_v1_type(self, catalog_type: str) -> str: + return self._V2_TO_V1_TYPE.get(catalog_type, catalog_type) + @property def _behavior_flags(self) -> list[BehaviorFlag]: return [ From f2f535e2843bebf0fc0335cd2e5d8bb521486be4 Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 11 May 2026 10:49:43 +0530 Subject: [PATCH 05/10] Update changelog for catalogs v2 rearchitecture --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d50bec96..8b959bc7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Features -- Declare `CATALOG_V2_CONFIGS` on `DatabricksAdapter` with `UnityDatabricksConfig` and `HiveMetastoreDatabricksConfig` to support parse-time validation of catalogs.yml v2 (requires `use_catalogs_v2` behavior flag in dbt-core) +- Add catalogs.yml v2 support via `Capability.CatalogsV2` and `_v2_to_v1_type` hook; move `use_uniform` × `file_format` cross-validation into `UnityCatalogIntegration.__init__` and file_format enum validation into `HiveMetastoreCatalogIntegration.__init__` (requires `use_catalogs_v2` flag in dbt-core and dbt-adapters >= 1.24) - Add `invocation_id` to the default query comment ([#1377](https://github.com/databricks/dbt-databricks/issues/1377)) ### Fixes From 0ad94f1ff33f4e6022d5edd72c90a47b7f8cb60d Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 11 May 2026 10:52:24 +0530 Subject: [PATCH 06/10] Simplify changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b959bc7c..3396ac210 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Features -- Add catalogs.yml v2 support via `Capability.CatalogsV2` and `_v2_to_v1_type` hook; move `use_uniform` × `file_format` cross-validation into `UnityCatalogIntegration.__init__` and file_format enum validation into `HiveMetastoreCatalogIntegration.__init__` (requires `use_catalogs_v2` flag in dbt-core and dbt-adapters >= 1.24) +- Add catalogs.yml v2 support (requires `use_catalogs_v2: true` in dbt-core) - Add `invocation_id` to the default query comment ([#1377](https://github.com/databricks/dbt-databricks/issues/1377)) ### Fixes From 0f630496ddc6028b4e2c92b49d0ae050b6611a58 Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 11 May 2026 10:59:34 +0530 Subject: [PATCH 07/10] Rewrite v2 catalog tests for new bridge architecture --- tests/unit/test_v2_catalog_configs.py | 118 +++++++++++++++----------- 1 file changed, 70 insertions(+), 48 deletions(-) diff --git a/tests/unit/test_v2_catalog_configs.py b/tests/unit/test_v2_catalog_configs.py index 814edf85d..890224c43 100644 --- a/tests/unit/test_v2_catalog_configs.py +++ b/tests/unit/test_v2_catalog_configs.py @@ -1,93 +1,115 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + import pytest -from dbt.adapters.databricks.catalogs._v2 import ( - HiveMetastoreDatabricksConfig, - UnityDatabricksConfig, +from dbt.adapters.capability import Capability, Support +from dbt.adapters.catalogs import CatalogIntegrationConfig +from dbt.adapters.databricks.catalogs import ( + HiveMetastoreCatalogIntegration, + UnityCatalogIntegration, ) from dbt.adapters.databricks.impl import DatabricksAdapter from dbt_common.exceptions import DbtValidationError -# ===== CATALOG_V2_CONFIGS class attribute ===== +@dataclass +class _Config: + """Minimal CatalogIntegrationConfig stub for testing __init__ validation.""" + + name: str = "test_cat" + catalog_type: str = "unity" + catalog_name: Optional[str] = None + table_format: Optional[str] = "iceberg" + external_volume: Optional[str] = None + file_format: Optional[str] = None + adapter_properties: Dict[str, Any] = field(default_factory=dict) + + +# ===== Adapter-level ===== + + +def test_catalogs_v2_capability_declared(): + cap = DatabricksAdapter._capabilities[Capability.CatalogsV2] + assert cap.support == Support.Full + +def test_v2_to_v1_type_unity(): + adapter = object.__new__(DatabricksAdapter) + assert adapter._v2_to_v1_type("unity") == "unity" -def test_unity_registered(): - assert DatabricksAdapter.CATALOG_V2_CONFIGS["unity"] is UnityDatabricksConfig +def test_v2_to_v1_type_hive_metastore(): + adapter = object.__new__(DatabricksAdapter) + assert adapter._v2_to_v1_type("hive_metastore") == "hive_metastore" -def test_hive_metastore_registered(): - assert ( - DatabricksAdapter.CATALOG_V2_CONFIGS["hive_metastore"] is HiveMetastoreDatabricksConfig - ) +def test_v2_to_v1_type_unknown_passthrough(): + adapter = object.__new__(DatabricksAdapter) + assert adapter._v2_to_v1_type("custom_type") == "custom_type" -# ===== UnityDatabricksConfig ===== + +# ===== UnityCatalogIntegration ===== def test_unity_parquet_without_uniform(): - cfg = UnityDatabricksConfig(file_format="parquet") - assert cfg.file_format == "parquet" + cfg = _Config(file_format="parquet") + integration = UnityCatalogIntegration(cfg) + assert integration.file_format == "parquet" def test_unity_delta_with_uniform(): - cfg = UnityDatabricksConfig(file_format="delta", use_uniform=True) - assert cfg.file_format == "delta" - assert cfg.use_uniform is True + cfg = _Config(file_format="delta", adapter_properties={"use_uniform": True}) + integration = UnityCatalogIntegration(cfg) + assert integration.file_format == "delta" def test_unity_with_location_root(): - cfg = UnityDatabricksConfig(file_format="parquet", location_root="/mnt/data") - assert cfg.location_root == "/mnt/data" + cfg = _Config(file_format="parquet", adapter_properties={"location_root": "/mnt/data"}) + integration = UnityCatalogIntegration(cfg) + assert integration.external_volume == "/mnt/data" def test_unity_delta_without_uniform_raises(): - with pytest.raises(DbtValidationError, match="file_format must be 'parquet'"): - UnityDatabricksConfig(file_format="delta") + cfg = _Config(file_format="delta") + with pytest.raises(DbtValidationError, match="use_uniform: false.*requires file_format: parquet"): + UnityCatalogIntegration(cfg) def test_unity_parquet_with_uniform_raises(): - with pytest.raises(DbtValidationError, match="file_format must be 'delta'"): - UnityDatabricksConfig(file_format="parquet", use_uniform=True) - - -def test_unity_blank_file_format_raises(): - with pytest.raises(DbtValidationError, match="file_format.*non-empty"): - UnityDatabricksConfig(file_format=" ") + cfg = _Config(file_format="parquet", adapter_properties={"use_uniform": True}) + with pytest.raises(DbtValidationError, match="use_uniform: true.*requires file_format: delta"): + UnityCatalogIntegration(cfg) def test_unity_blank_location_root_raises(): - with pytest.raises(DbtValidationError, match="location_root.*blank"): - UnityDatabricksConfig(file_format="parquet", location_root=" ") - + cfg = _Config(file_format="parquet", adapter_properties={"location_root": " "}) + with pytest.raises(DbtValidationError, match="location_root cannot be blank"): + UnityCatalogIntegration(cfg) -def test_unity_rejects_unknown_keys(): - with pytest.raises(Exception, match="Additional properties"): - UnityDatabricksConfig.validate({"file_format": "parquet", "bogus": True}) - -# ===== HiveMetastoreDatabricksConfig ===== +# ===== HiveMetastoreCatalogIntegration ===== def test_hive_delta_valid(): - cfg = HiveMetastoreDatabricksConfig(file_format="delta") - assert cfg.file_format == "delta" + cfg = _Config(catalog_type="hive_metastore", file_format="delta") + integration = HiveMetastoreCatalogIntegration(cfg) + assert integration.file_format == "delta" def test_hive_parquet_valid(): - cfg = HiveMetastoreDatabricksConfig(file_format="parquet") - assert cfg.file_format == "parquet" + cfg = _Config(catalog_type="hive_metastore", file_format="parquet") + integration = HiveMetastoreCatalogIntegration(cfg) + assert integration.file_format == "parquet" def test_hive_hudi_valid(): - cfg = HiveMetastoreDatabricksConfig(file_format="hudi") - assert cfg.file_format == "hudi" + cfg = _Config(catalog_type="hive_metastore", file_format="hudi") + integration = HiveMetastoreCatalogIntegration(cfg) + assert integration.file_format == "hudi" def test_hive_invalid_file_format_raises(): - with pytest.raises(DbtValidationError, match="file_format must be one of"): - HiveMetastoreDatabricksConfig(file_format="avro") - - -def test_hive_rejects_unknown_keys(): - with pytest.raises(Exception, match="Additional properties"): - HiveMetastoreDatabricksConfig.validate({"file_format": "delta", "extra": "bad"}) + cfg = _Config(catalog_type="hive_metastore", file_format="avro") + with pytest.raises(DbtValidationError, match="file_format"): + HiveMetastoreCatalogIntegration(cfg) \ No newline at end of file From fda34d68a080b1cdd2e753fdc6e8e7ff5ce354b5 Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 11 May 2026 11:00:24 +0530 Subject: [PATCH 08/10] Rename test_v2_catalog_configs.py to test_catalogs_v2.py --- tests/unit/{test_v2_catalog_configs.py => test_catalogs_v2.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/unit/{test_v2_catalog_configs.py => test_catalogs_v2.py} (100%) diff --git a/tests/unit/test_v2_catalog_configs.py b/tests/unit/test_catalogs_v2.py similarity index 100% rename from tests/unit/test_v2_catalog_configs.py rename to tests/unit/test_catalogs_v2.py From 80e9d188bfffdffbb6c51e90bb2ca5d7cabfd6a9 Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 11 May 2026 11:05:15 +0530 Subject: [PATCH 09/10] Fix location_root blank check, remove unused import, add empty string test --- dbt/adapters/databricks/catalogs/_unity.py | 3 ++- tests/unit/test_catalogs_v2.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/databricks/catalogs/_unity.py b/dbt/adapters/databricks/catalogs/_unity.py index b68b68c12..9f55328bf 100644 --- a/dbt/adapters/databricks/catalogs/_unity.py +++ b/dbt/adapters/databricks/catalogs/_unity.py @@ -14,7 +14,8 @@ class UnityCatalogIntegration(CatalogIntegration): def __init__(self, config: CatalogIntegrationConfig) -> None: super().__init__(config) - if location_root := config.adapter_properties.get("location_root"): + location_root = config.adapter_properties.get("location_root") + if location_root is not None: if not str(location_root).strip(): raise DbtValidationError( f"Catalog '{config.name}' unity/databricks location_root cannot be blank" diff --git a/tests/unit/test_catalogs_v2.py b/tests/unit/test_catalogs_v2.py index 890224c43..26b6e1fe5 100644 --- a/tests/unit/test_catalogs_v2.py +++ b/tests/unit/test_catalogs_v2.py @@ -4,7 +4,6 @@ import pytest from dbt.adapters.capability import Capability, Support -from dbt.adapters.catalogs import CatalogIntegrationConfig from dbt.adapters.databricks.catalogs import ( HiveMetastoreCatalogIntegration, UnityCatalogIntegration, @@ -88,6 +87,12 @@ def test_unity_blank_location_root_raises(): UnityCatalogIntegration(cfg) +def test_unity_empty_location_root_raises(): + cfg = _Config(file_format="parquet", adapter_properties={"location_root": ""}) + with pytest.raises(DbtValidationError, match="location_root cannot be blank"): + UnityCatalogIntegration(cfg) + + # ===== HiveMetastoreCatalogIntegration ===== From fae1603204587be68f387e1d8fc89b8f341674dc Mon Sep 17 00:00:00 2001 From: Aahel Guha Date: Mon, 25 May 2026 12:34:35 +0530 Subject: [PATCH 10/10] minor fmt fix --- dbt/adapters/databricks/catalogs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/databricks/catalogs/__init__.py b/dbt/adapters/databricks/catalogs/__init__.py index c434823a6..a6f1bed39 100644 --- a/dbt/adapters/databricks/catalogs/__init__.py +++ b/dbt/adapters/databricks/catalogs/__init__.py @@ -6,4 +6,4 @@ "DatabricksCatalogRelation", "HiveMetastoreCatalogIntegration", "UnityCatalogIntegration", -] \ No newline at end of file +]