fix: add insert-by-name support to microbatch and replace_where strategies (#1338)

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Fixes
 
+- Fix column order mismatch in microbatch and replace_where incremental strategies by using INSERT BY NAME syntax ([#1338](https://github.com/databricks/dbt-databricks/issues/1338))
 - Fix missing `optimize()` call in table v2 materialization path ([#1345](https://github.com/databricks/dbt-databricks/pull/1345))
 - Fix catalog names with special characters (e.g., hyphens) not being quoted in `SHOW SCHEMAS` commands, causing `INVALID_IDENTIFIER` errors ([#1325](https://github.com/databricks/dbt-databricks/issues/1325))
 - Fix liquid clustering rendering on streaming table materialization [#1330](https://github.com/databricks/dbt-databricks/pull/1330)

dbt/include/databricks/macros/materializations/incremental/strategies.sql

@@ -121,6 +121,7 @@
   {%- set predicates = args_dict['incremental_predicates'] -%}
   {%- set target_relation = args_dict['target_relation'] -%}
   {%- set temp_relation = args_dict['temp_relation'] -%}
+  {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}
 INSERT INTO {{ target_relation.render() }}
 {%- if predicates %}
   {%- if predicates is sequence and predicates is not string %}
@@ -129,6 +130,7 @@ INSERT INTO {{ target_relation.render() }}
  REPLACE WHERE {{ predicates }}
   {%- endif %}
 {%- endif %}
+{%- if has_insert_by_name %} BY NAME{% endif %}
  TABLE {{ temp_relation.render() }}
 {%- endmacro %}
 

tests/unit/macros/materializations/incremental/test_microbatch_macros.py

@@ -14,6 +14,17 @@ def template_name(self) -> str:
     def macro_folders_to_load(self) -> list:
         return ["macros", "macros/materializations/incremental"]
 
+    @pytest.fixture(autouse=True)
+    def setup_mock_capability(self, context):
+        """Mock the adapter's has_dbr_capability to support insert_by_name by default"""
+
+        def has_dbr_capability_side_effect(capability_name):
+            if capability_name == "insert_by_name":
+                return True  # Default to DBR 12.2+
+            return False
+
+        context["adapter"].has_dbr_capability = Mock(side_effect=has_dbr_capability_side_effect)
+
     @pytest.fixture
     def mock_temp(self):
         relation = Mock()
@@ -51,7 +62,8 @@ def test_databricks__get_incremental_microbatch_sql_with_start_and_end(
         expected = """
         insert into `some_database`.`some_schema`.`some_table`
         replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00'
-            and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' table `temp_table`
+            and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00'
+            by name table `temp_table`
 """
 
         self.assert_sql_equal(result, expected)
@@ -75,7 +87,8 @@ def test_databricks__get_incremental_microbatch_sql_with_start_only(
 
         expected = """
         insert into `some_database`.`some_schema`.`some_table`
-        replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' table `temp_table`
+        replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00'
+            by name table `temp_table`
 """
 
         self.assert_sql_equal(result, expected)
@@ -99,7 +112,8 @@ def test_databricks__get_incremental_microbatch_sql_with_end_only(
 
         expected = """
         insert into `some_database`.`some_schema`.`some_table`
-        replace where cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' table `temp_table`
+        replace where cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00'
+            by name table `temp_table`
 """
 
         self.assert_sql_equal(result, expected)
@@ -127,7 +141,8 @@ def test_databricks__get_incremental_microbatch_sql_with_existing_predicates(
         insert into `some_database`.`some_schema`.`some_table`
         replace where col1 = 'value'
             and col2 > 100 and cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00'
-            and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' table `temp_table`
+            and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00'
+            by name table `temp_table`
 """
 
         self.assert_sql_equal(result, expected)
@@ -151,7 +166,37 @@ def test_databricks__get_incremental_microbatch_sql_without_start_or_end(
 
         expected = """
         insert into `some_database`.`some_schema`.`some_table`
-        table `temp_table`
+        by name table `temp_table`
+"""
+
+        self.assert_sql_equal(result, expected)
+
+    def test_databricks__get_incremental_microbatch_sql_without_insert_by_name(
+        self, template_bundle, context, config, mock_arg_dict
+    ):
+        """Test that BY NAME is omitted when the DBR version doesn't support insert_by_name"""
+
+        context["adapter"].has_dbr_capability = Mock(return_value=False)
+
+        context["model"] = {"config": {"event_time": "event_timestamp"}}
+        config["__dbt_internal_microbatch_event_time_start"] = "2023-01-01 00:00:00"
+        config["__dbt_internal_microbatch_event_time_end"] = "2023-01-02 00:00:00"
+
+        context["return"] = Mock()
+
+        self.run_macro_raw(
+            template_bundle.template,
+            "databricks__get_incremental_microbatch_sql",
+            mock_arg_dict,
+        )
+
+        result = context["return"].call_args[0][0]
+
+        expected = """
+        insert into `some_database`.`some_schema`.`some_table`
+        replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00'
+            and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00'
+            table `temp_table`
 """
 
         self.assert_sql_equal(result, expected)

tests/unit/macros/materializations/incremental/test_replace_where_macros.py

@@ -14,6 +14,17 @@ def template_name(self) -> str:
     def macro_folders_to_load(self) -> list:
         return ["macros", "macros/materializations/incremental"]
 
+    @pytest.fixture(autouse=True)
+    def setup_mock_capability(self, context):
+        """Mock the adapter's has_dbr_capability to support insert_by_name by default"""
+
+        def has_dbr_capability_side_effect(capability_name):
+            if capability_name == "insert_by_name":
+                return True  # Default to DBR 12.2+
+            return False
+
+        context["adapter"].has_dbr_capability = Mock(side_effect=has_dbr_capability_side_effect)
+
     @pytest.fixture
     def mock_relations(self):
         target_relation = Mock()
@@ -38,7 +49,7 @@ def test_get_replace_where_sql_with_string_predicate(self, template_bundle, mock
         expected = """
         INSERT INTO schema.target_table
         REPLACE WHERE date_col > '2023-01-01'
-        TABLE schema.temp_table
+        BY NAME TABLE schema.temp_table
         """
 
         self.assert_sql_equal(result, expected)
@@ -60,7 +71,7 @@ def test_get_replace_where_sql_with_predicate_list(self, template_bundle, mock_r
         expected = """
         INSERT INTO schema.target_table
         REPLACE WHERE date_col > '2023-01-01' and another_col != 'value'
-        TABLE schema.temp_table
+        BY NAME TABLE schema.temp_table
         """
 
         self.assert_sql_equal(result, expected)
@@ -78,7 +89,7 @@ def test_get_replace_where_sql_without_predicates(self, template_bundle, mock_re
 
         expected = """
         INSERT INTO schema.target_table
-        TABLE schema.temp_table
+        BY NAME TABLE schema.temp_table
         """
 
         self.assert_sql_equal(result, expected)
@@ -96,7 +107,7 @@ def test_get_replace_where_sql_with_empty_predicate_list(self, template_bundle,
 
         expected = """
         INSERT INTO schema.target_table
-        TABLE schema.temp_table
+        BY NAME TABLE schema.temp_table
         """
 
         self.assert_sql_equal(result, expected)
@@ -119,13 +130,37 @@ def test_get_replace_where_sql_with_complex_predicates(self, template_bundle, mo
         expected = """
         INSERT INTO schema.target_table
         REPLACE WHERE date_col BETWEEN '2023-01-01' AND '2023-01-31' and status IN ('active', 'pending') and amount > 1000
-        TABLE schema.temp_table
+        BY NAME TABLE schema.temp_table
         """  # noqa
 
         self.assert_sql_equal(result, expected)
 
     def test_get_replace_where_sql__by_name_with_predicates(self, template_bundle, mock_relations):
-        """Test that get_replace_where_sql does not use BY NAME (removed as of recent change)"""
+        """Test that get_replace_where_sql uses BY NAME when insert_by_name capability is available"""
+        target_relation, temp_relation = mock_relations
+
+        args_dict = {
+            "target_relation": target_relation,
+            "temp_relation": temp_relation,
+            "incremental_predicates": "date_col > '2023-01-01'",
+        }
+
+        result = self.run_macro(template_bundle.template, "get_replace_where_sql", args_dict)
+
+        expected = """
+        INSERT INTO schema.target_table
+        REPLACE WHERE date_col > '2023-01-01'
+        BY NAME TABLE schema.temp_table
+        """
+
+        self.assert_sql_equal(result, expected)
+
+    def test_get_replace_where_sql__without_insert_by_name(
+        self, template_bundle, context, mock_relations
+    ):
+        """Test that BY NAME is omitted when the DBR version doesn't support insert_by_name"""
+        context["adapter"].has_dbr_capability = Mock(return_value=False)
+
         target_relation, temp_relation = mock_relations
 
         args_dict = {