From e32a6181add1158af9266571df2df831f84ba2a6 Mon Sep 17 00:00:00 2001 From: aarushisingh04 Date: Sun, 8 Mar 2026 01:03:47 +0530 Subject: [PATCH 1/6] fix: add insert-by-name support to microbatch and replace_where strategies (#1338) Signed-off-by: aarushisingh04 --- CHANGELOG.md | 1 + .../incremental/strategies.sql | 2 + .../incremental/test_microbatch_macros.py | 55 +++++++++++++++++-- .../incremental/test_replace_where_macros.py | 47 ++++++++++++++-- 4 files changed, 94 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2480a1cab..34f1ad387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Fixes +- Fix column order mismatch in microbatch and replace_where incremental strategies by using INSERT BY NAME syntax ([#1338](https://github.com/databricks/dbt-databricks/issues/1338)) - Fix missing `optimize()` call in table v2 materialization path ([#1345](https://github.com/databricks/dbt-databricks/pull/1345)) - Fix catalog names with special characters (e.g., hyphens) not being quoted in `SHOW SCHEMAS` commands, causing `INVALID_IDENTIFIER` errors ([#1325](https://github.com/databricks/dbt-databricks/issues/1325)) - Fix liquid clustering rendering on streaming table materialization [#1330](https://github.com/databricks/dbt-databricks/pull/1330) diff --git a/dbt/include/databricks/macros/materializations/incremental/strategies.sql b/dbt/include/databricks/macros/materializations/incremental/strategies.sql index c1184b60d..099cdffbd 100644 --- a/dbt/include/databricks/macros/materializations/incremental/strategies.sql +++ b/dbt/include/databricks/macros/materializations/incremental/strategies.sql @@ -121,6 +121,7 @@ {%- set predicates = args_dict['incremental_predicates'] -%} {%- set target_relation = args_dict['target_relation'] -%} {%- set temp_relation = args_dict['temp_relation'] -%} + {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%} INSERT INTO {{ target_relation.render() }} {%- if predicates %} {%- if predicates is sequence and predicates is not string %} @@ -129,6 +130,7 @@ INSERT INTO {{ target_relation.render() }} REPLACE WHERE {{ predicates }} {%- endif %} {%- endif %} +{%- if has_insert_by_name %} BY NAME{% endif %} TABLE {{ temp_relation.render() }} {%- endmacro %} diff --git a/tests/unit/macros/materializations/incremental/test_microbatch_macros.py b/tests/unit/macros/materializations/incremental/test_microbatch_macros.py index 390b9a868..f7ba8e838 100644 --- a/tests/unit/macros/materializations/incremental/test_microbatch_macros.py +++ b/tests/unit/macros/materializations/incremental/test_microbatch_macros.py @@ -14,6 +14,17 @@ def template_name(self) -> str: def macro_folders_to_load(self) -> list: return ["macros", "macros/materializations/incremental"] + @pytest.fixture(autouse=True) + def setup_mock_capability(self, context): + """Mock the adapter's has_dbr_capability to support insert_by_name by default""" + + def has_dbr_capability_side_effect(capability_name): + if capability_name == "insert_by_name": + return True # Default to DBR 12.2+ + return False + + context["adapter"].has_dbr_capability = Mock(side_effect=has_dbr_capability_side_effect) + @pytest.fixture def mock_temp(self): relation = Mock() @@ -51,7 +62,8 @@ def test_databricks__get_incremental_microbatch_sql_with_start_and_end( expected = """ insert into `some_database`.`some_schema`.`some_table` replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' - and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' table `temp_table` + and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' + by name table `temp_table` """ self.assert_sql_equal(result, expected) @@ -75,7 +87,8 @@ def test_databricks__get_incremental_microbatch_sql_with_start_only( expected = """ insert into `some_database`.`some_schema`.`some_table` - replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' table `temp_table` + replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' + by name table `temp_table` """ self.assert_sql_equal(result, expected) @@ -99,7 +112,8 @@ def test_databricks__get_incremental_microbatch_sql_with_end_only( expected = """ insert into `some_database`.`some_schema`.`some_table` - replace where cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' table `temp_table` + replace where cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' + by name table `temp_table` """ self.assert_sql_equal(result, expected) @@ -127,7 +141,8 @@ def test_databricks__get_incremental_microbatch_sql_with_existing_predicates( insert into `some_database`.`some_schema`.`some_table` replace where col1 = 'value' and col2 > 100 and cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' - and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' table `temp_table` + and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' + by name table `temp_table` """ self.assert_sql_equal(result, expected) @@ -151,7 +166,37 @@ def test_databricks__get_incremental_microbatch_sql_without_start_or_end( expected = """ insert into `some_database`.`some_schema`.`some_table` - table `temp_table` + by name table `temp_table` +""" + + self.assert_sql_equal(result, expected) + + def test_databricks__get_incremental_microbatch_sql_without_insert_by_name( + self, template_bundle, context, config, mock_arg_dict + ): + """Test that BY NAME is omitted when the DBR version doesn't support insert_by_name""" + + context["adapter"].has_dbr_capability = Mock(return_value=False) + + context["model"] = {"config": {"event_time": "event_timestamp"}} + config["__dbt_internal_microbatch_event_time_start"] = "2023-01-01 00:00:00" + config["__dbt_internal_microbatch_event_time_end"] = "2023-01-02 00:00:00" + + context["return"] = Mock() + + self.run_macro_raw( + template_bundle.template, + "databricks__get_incremental_microbatch_sql", + mock_arg_dict, + ) + + result = context["return"].call_args[0][0] + + expected = """ + insert into `some_database`.`some_schema`.`some_table` + replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' + and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' + table `temp_table` """ self.assert_sql_equal(result, expected) diff --git a/tests/unit/macros/materializations/incremental/test_replace_where_macros.py b/tests/unit/macros/materializations/incremental/test_replace_where_macros.py index 1c984969f..eede1de61 100644 --- a/tests/unit/macros/materializations/incremental/test_replace_where_macros.py +++ b/tests/unit/macros/materializations/incremental/test_replace_where_macros.py @@ -14,6 +14,17 @@ def template_name(self) -> str: def macro_folders_to_load(self) -> list: return ["macros", "macros/materializations/incremental"] + @pytest.fixture(autouse=True) + def setup_mock_capability(self, context): + """Mock the adapter's has_dbr_capability to support insert_by_name by default""" + + def has_dbr_capability_side_effect(capability_name): + if capability_name == "insert_by_name": + return True # Default to DBR 12.2+ + return False + + context["adapter"].has_dbr_capability = Mock(side_effect=has_dbr_capability_side_effect) + @pytest.fixture def mock_relations(self): target_relation = Mock() @@ -38,7 +49,7 @@ def test_get_replace_where_sql_with_string_predicate(self, template_bundle, mock expected = """ INSERT INTO schema.target_table REPLACE WHERE date_col > '2023-01-01' - TABLE schema.temp_table + BY NAME TABLE schema.temp_table """ self.assert_sql_equal(result, expected) @@ -60,7 +71,7 @@ def test_get_replace_where_sql_with_predicate_list(self, template_bundle, mock_r expected = """ INSERT INTO schema.target_table REPLACE WHERE date_col > '2023-01-01' and another_col != 'value' - TABLE schema.temp_table + BY NAME TABLE schema.temp_table """ self.assert_sql_equal(result, expected) @@ -78,7 +89,7 @@ def test_get_replace_where_sql_without_predicates(self, template_bundle, mock_re expected = """ INSERT INTO schema.target_table - TABLE schema.temp_table + BY NAME TABLE schema.temp_table """ self.assert_sql_equal(result, expected) @@ -96,7 +107,7 @@ def test_get_replace_where_sql_with_empty_predicate_list(self, template_bundle, expected = """ INSERT INTO schema.target_table - TABLE schema.temp_table + BY NAME TABLE schema.temp_table """ self.assert_sql_equal(result, expected) @@ -119,13 +130,37 @@ def test_get_replace_where_sql_with_complex_predicates(self, template_bundle, mo expected = """ INSERT INTO schema.target_table REPLACE WHERE date_col BETWEEN '2023-01-01' AND '2023-01-31' and status IN ('active', 'pending') and amount > 1000 - TABLE schema.temp_table + BY NAME TABLE schema.temp_table """ # noqa self.assert_sql_equal(result, expected) def test_get_replace_where_sql__by_name_with_predicates(self, template_bundle, mock_relations): - """Test that get_replace_where_sql does not use BY NAME (removed as of recent change)""" + """Test that get_replace_where_sql uses BY NAME when insert_by_name capability is available""" + target_relation, temp_relation = mock_relations + + args_dict = { + "target_relation": target_relation, + "temp_relation": temp_relation, + "incremental_predicates": "date_col > '2023-01-01'", + } + + result = self.run_macro(template_bundle.template, "get_replace_where_sql", args_dict) + + expected = """ + INSERT INTO schema.target_table + REPLACE WHERE date_col > '2023-01-01' + BY NAME TABLE schema.temp_table + """ + + self.assert_sql_equal(result, expected) + + def test_get_replace_where_sql__without_insert_by_name( + self, template_bundle, context, mock_relations + ): + """Test that BY NAME is omitted when the DBR version doesn't support insert_by_name""" + context["adapter"].has_dbr_capability = Mock(return_value=False) + target_relation, temp_relation = mock_relations args_dict = { From 9c2ddd70cf2db6de63e2a85f67d93bd82632b5c9 Mon Sep 17 00:00:00 2001 From: aarushisingh04 Date: Tue, 10 Mar 2026 15:56:28 +0530 Subject: [PATCH 2/6] test: add microbatch insert-by-name functional test and remove redundant unit test Signed-off-by: aarushisingh04 --- .../functional/adapter/microbatch/fixtures.py | 44 ++++++++++++ .../test_microbatch_insert_by_name.py | 69 +++++++++++++++++++ .../incremental/test_replace_where_macros.py | 20 ------ 3 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py diff --git a/tests/functional/adapter/microbatch/fixtures.py b/tests/functional/adapter/microbatch/fixtures.py index 9a6dc900d..e271cd5d3 100644 --- a/tests/functional/adapter/microbatch/fixtures.py +++ b/tests/functional/adapter/microbatch/fixtures.py @@ -14,3 +14,47 @@ - name: event_time description: "Timestamp of the event" """ + +microbatch_seeds_csv = """ +id,event_time,amount +1,2023-01-01,100 +2,2023-01-01,200 +3,2023-01-02,300 +""".strip() + +# Initial model: columns in (id, event_time, amount) order +microbatch_model_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='microbatch', + event_time='event_time', + begin='2023-01-01', + batch_size='day' +) }} +select id, event_time, amount from {{ ref('microbatch_seeds') }} +where {{ incremental_predicate('event_time') }} +""" + +# Reordered model: columns in (amount, id, event_time) order — this is the key scenario +# Without BY NAME, positional INSERT would silently corrupt data here +microbatch_model_reordered_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='microbatch', + event_time='event_time', + begin='2023-01-01', + batch_size='day' +) }} +select amount, id, event_time from {{ ref('microbatch_seeds') }} +where {{ incremental_predicate('event_time') }} +""" + +schema_yml = """ +version: 2 +models: + - name: microbatch_model + columns: + - name: id + - name: event_time + - name: amount +""" diff --git a/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py new file mode 100644 index 000000000..5b430d19a --- /dev/null +++ b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py @@ -0,0 +1,69 @@ +import pytest +from dbt.tests import util + +from tests.functional.adapter.microbatch.fixtures import ( + microbatch_model_sql, + microbatch_model_reordered_sql, + microbatch_seeds_csv, + schema_yml, +) + +class TestMicrobatchInsertByName: + """ + Verifies that the microbatch strategy uses INSERT BY NAME, which preserves + data integrity when column order changes between batches (DBR 12.2+). + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "microbatch_model.sql": microbatch_model_sql, + "schema.yml": schema_yml, + } + + @pytest.fixture(scope="class") + def seeds(self): + return { + "microbatch_seeds.csv": microbatch_seeds_csv, + } + + def test_microbatch_insert_by_name(self, project): + # Initial run — creates the table with original column order + util.run_dbt(["seed"]) + util.run_dbt(["run"]) + + # Verify initial data using direct SQL comparison and fully qualified name + actual_initial = project.run_sql( + f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", + fetch="all" + ) + assert len(actual_initial) == 3 + assert actual_initial[0] == (1, 100) + assert actual_initial[1] == (2, 200) + assert actual_initial[2] == (3, 300) + + # Simulate column order change by swapping to reordered model + # Add new data to ensure an incremental batch is processed + new_seeds_csv = microbatch_seeds_csv.strip() + "\n4,2023-01-03,400\n" + util.write_file(new_seeds_csv, str(project.project_root) + "/seeds/microbatch_seeds.csv") + + util.write_file(microbatch_model_reordered_sql, str(project.project_root) + "/models/microbatch_model.sql") + + # Second run — should correctly insert by name, not position + util.run_dbt(["seed"]) + util.run_dbt(["run"]) + + # Verify row count and data integrity are correct + actual_final = project.run_sql( + f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", + fetch="all" + ) + + assert len(actual_final) == 4 + assert actual_final[0] == (1, 100) + assert actual_final[1] == (2, 200) + assert actual_final[2] == (3, 300) + + # The critical assertion: amount must be 400, not 4 (which id=4 would produce positionally) + # Without BY NAME, the data from the reordered select would map into the wrong columns. + assert actual_final[3] == (4, 400) diff --git a/tests/unit/macros/materializations/incremental/test_replace_where_macros.py b/tests/unit/macros/materializations/incremental/test_replace_where_macros.py index eede1de61..37bc13a8c 100644 --- a/tests/unit/macros/materializations/incremental/test_replace_where_macros.py +++ b/tests/unit/macros/materializations/incremental/test_replace_where_macros.py @@ -135,26 +135,6 @@ def test_get_replace_where_sql_with_complex_predicates(self, template_bundle, mo self.assert_sql_equal(result, expected) - def test_get_replace_where_sql__by_name_with_predicates(self, template_bundle, mock_relations): - """Test that get_replace_where_sql uses BY NAME when insert_by_name capability is available""" - target_relation, temp_relation = mock_relations - - args_dict = { - "target_relation": target_relation, - "temp_relation": temp_relation, - "incremental_predicates": "date_col > '2023-01-01'", - } - - result = self.run_macro(template_bundle.template, "get_replace_where_sql", args_dict) - - expected = """ - INSERT INTO schema.target_table - REPLACE WHERE date_col > '2023-01-01' - BY NAME TABLE schema.temp_table - """ - - self.assert_sql_equal(result, expected) - def test_get_replace_where_sql__without_insert_by_name( self, template_bundle, context, mock_relations ): From f224ff73168d9d484e9d88384be3dd70d07fc62a Mon Sep 17 00:00:00 2001 From: aarushisingh04 Date: Thu, 12 Mar 2026 17:48:15 +0530 Subject: [PATCH 3/6] minor: ran pre-commit Signed-off-by: aarushisingh04 --- .../test_microbatch_insert_by_name.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py index 5b430d19a..747fd9f2a 100644 --- a/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py +++ b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py @@ -2,12 +2,13 @@ from dbt.tests import util from tests.functional.adapter.microbatch.fixtures import ( - microbatch_model_sql, microbatch_model_reordered_sql, + microbatch_model_sql, microbatch_seeds_csv, schema_yml, ) + class TestMicrobatchInsertByName: """ Verifies that the microbatch strategy uses INSERT BY NAME, which preserves @@ -34,8 +35,8 @@ def test_microbatch_insert_by_name(self, project): # Verify initial data using direct SQL comparison and fully qualified name actual_initial = project.run_sql( - f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", - fetch="all" + f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", + fetch="all", ) assert len(actual_initial) == 3 assert actual_initial[0] == (1, 100) @@ -46,8 +47,11 @@ def test_microbatch_insert_by_name(self, project): # Add new data to ensure an incremental batch is processed new_seeds_csv = microbatch_seeds_csv.strip() + "\n4,2023-01-03,400\n" util.write_file(new_seeds_csv, str(project.project_root) + "/seeds/microbatch_seeds.csv") - - util.write_file(microbatch_model_reordered_sql, str(project.project_root) + "/models/microbatch_model.sql") + + util.write_file( + microbatch_model_reordered_sql, + str(project.project_root) + "/models/microbatch_model.sql", + ) # Second run — should correctly insert by name, not position util.run_dbt(["seed"]) @@ -55,15 +59,15 @@ def test_microbatch_insert_by_name(self, project): # Verify row count and data integrity are correct actual_final = project.run_sql( - f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", - fetch="all" + f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", + fetch="all", ) - + assert len(actual_final) == 4 assert actual_final[0] == (1, 100) assert actual_final[1] == (2, 200) assert actual_final[2] == (3, 300) - + # The critical assertion: amount must be 400, not 4 (which id=4 would produce positionally) # Without BY NAME, the data from the reordered select would map into the wrong columns. assert actual_final[3] == (4, 400) From 77b52170db4914e9e5070ccda073e19cf507d68d Mon Sep 17 00:00:00 2001 From: aarushisingh04 Date: Thu, 19 Mar 2026 11:15:49 +0530 Subject: [PATCH 4/6] minor: ran pre-commit Signed-off-by: aarushisingh04 --- .../adapter/microbatch/test_microbatch_insert_by_name.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py index 747fd9f2a..7c25ad99b 100644 --- a/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py +++ b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py @@ -35,7 +35,8 @@ def test_microbatch_insert_by_name(self, project): # Verify initial data using direct SQL comparison and fully qualified name actual_initial = project.run_sql( - f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", + f"select id, amount from {project.database}.{project.test_schema}.microbatch_model " + "order by id", fetch="all", ) assert len(actual_initial) == 3 @@ -59,7 +60,8 @@ def test_microbatch_insert_by_name(self, project): # Verify row count and data integrity are correct actual_final = project.run_sql( - f"select id, amount from {project.database}.{project.test_schema}.microbatch_model order by id", + f"select id, amount from {project.database}.{project.test_schema}.microbatch_model " + "order by id", fetch="all", ) From 519e97e57477597534785ce9c254bc98f3219ef6 Mon Sep 17 00:00:00 2001 From: aarushisingh04 Date: Thu, 19 Mar 2026 13:28:14 +0530 Subject: [PATCH 5/6] fix: sql syntax error at 'by' Signed-off-by: aarushisingh04 --- .../materializations/incremental/strategies.sql | 2 +- .../incremental/test_microbatch_macros.py | 16 ++++++++-------- .../incremental/test_replace_where_macros.py | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/dbt/include/databricks/macros/materializations/incremental/strategies.sql b/dbt/include/databricks/macros/materializations/incremental/strategies.sql index 099cdffbd..cd2805096 100644 --- a/dbt/include/databricks/macros/materializations/incremental/strategies.sql +++ b/dbt/include/databricks/macros/materializations/incremental/strategies.sql @@ -123,6 +123,7 @@ {%- set temp_relation = args_dict['temp_relation'] -%} {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%} INSERT INTO {{ target_relation.render() }} +{%- if has_insert_by_name %} BY NAME{% endif %} {%- if predicates %} {%- if predicates is sequence and predicates is not string %} REPLACE WHERE {{ predicates | join(' and ') }} @@ -130,7 +131,6 @@ INSERT INTO {{ target_relation.render() }} REPLACE WHERE {{ predicates }} {%- endif %} {%- endif %} -{%- if has_insert_by_name %} BY NAME{% endif %} TABLE {{ temp_relation.render() }} {%- endmacro %} diff --git a/tests/unit/macros/materializations/incremental/test_microbatch_macros.py b/tests/unit/macros/materializations/incremental/test_microbatch_macros.py index f7ba8e838..183c2c1a0 100644 --- a/tests/unit/macros/materializations/incremental/test_microbatch_macros.py +++ b/tests/unit/macros/materializations/incremental/test_microbatch_macros.py @@ -61,9 +61,9 @@ def test_databricks__get_incremental_microbatch_sql_with_start_and_end( expected = """ insert into `some_database`.`some_schema`.`some_table` - replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' + by name replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' - by name table `temp_table` + table `temp_table` """ self.assert_sql_equal(result, expected) @@ -87,8 +87,8 @@ def test_databricks__get_incremental_microbatch_sql_with_start_only( expected = """ insert into `some_database`.`some_schema`.`some_table` - replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' - by name table `temp_table` + by name replace where cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' + table `temp_table` """ self.assert_sql_equal(result, expected) @@ -112,8 +112,8 @@ def test_databricks__get_incremental_microbatch_sql_with_end_only( expected = """ insert into `some_database`.`some_schema`.`some_table` - replace where cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' - by name table `temp_table` + by name replace where cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' + table `temp_table` """ self.assert_sql_equal(result, expected) @@ -139,10 +139,10 @@ def test_databricks__get_incremental_microbatch_sql_with_existing_predicates( expected = """ insert into `some_database`.`some_schema`.`some_table` - replace where col1 = 'value' + by name replace where col1 = 'value' and col2 > 100 and cast(event_timestamp as TIMESTAMP) >= '2023-01-01 00:00:00' and cast(event_timestamp as TIMESTAMP) < '2023-01-02 00:00:00' - by name table `temp_table` + table `temp_table` """ self.assert_sql_equal(result, expected) diff --git a/tests/unit/macros/materializations/incremental/test_replace_where_macros.py b/tests/unit/macros/materializations/incremental/test_replace_where_macros.py index 37bc13a8c..54dcb7b6a 100644 --- a/tests/unit/macros/materializations/incremental/test_replace_where_macros.py +++ b/tests/unit/macros/materializations/incremental/test_replace_where_macros.py @@ -48,8 +48,8 @@ def test_get_replace_where_sql_with_string_predicate(self, template_bundle, mock expected = """ INSERT INTO schema.target_table - REPLACE WHERE date_col > '2023-01-01' - BY NAME TABLE schema.temp_table + BY NAME REPLACE WHERE date_col > '2023-01-01' + TABLE schema.temp_table """ self.assert_sql_equal(result, expected) @@ -70,8 +70,8 @@ def test_get_replace_where_sql_with_predicate_list(self, template_bundle, mock_r expected = """ INSERT INTO schema.target_table - REPLACE WHERE date_col > '2023-01-01' and another_col != 'value' - BY NAME TABLE schema.temp_table + BY NAME REPLACE WHERE date_col > '2023-01-01' and another_col != 'value' + TABLE schema.temp_table """ self.assert_sql_equal(result, expected) @@ -129,8 +129,8 @@ def test_get_replace_where_sql_with_complex_predicates(self, template_bundle, mo expected = """ INSERT INTO schema.target_table - REPLACE WHERE date_col BETWEEN '2023-01-01' AND '2023-01-31' and status IN ('active', 'pending') and amount > 1000 - BY NAME TABLE schema.temp_table + BY NAME REPLACE WHERE date_col BETWEEN '2023-01-01' AND '2023-01-31' and status IN ('active', 'pending') and amount > 1000 + TABLE schema.temp_table """ # noqa self.assert_sql_equal(result, expected) From 8d3b62c90511a3294f3901c7282f6c42490687e5 Mon Sep 17 00:00:00 2001 From: aarushisingh04 Date: Sun, 22 Mar 2026 01:31:27 +0530 Subject: [PATCH 6/6] fix: rewrite functional test for replace_where column order Signed-off-by: aarushisingh04 --- .../functional/adapter/microbatch/fixtures.py | 2 - .../test_microbatch_insert_by_name.py | 81 ++++++++++++------- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/tests/functional/adapter/microbatch/fixtures.py b/tests/functional/adapter/microbatch/fixtures.py index e271cd5d3..c09ca9bf3 100644 --- a/tests/functional/adapter/microbatch/fixtures.py +++ b/tests/functional/adapter/microbatch/fixtures.py @@ -32,7 +32,6 @@ batch_size='day' ) }} select id, event_time, amount from {{ ref('microbatch_seeds') }} -where {{ incremental_predicate('event_time') }} """ # Reordered model: columns in (amount, id, event_time) order — this is the key scenario @@ -46,7 +45,6 @@ batch_size='day' ) }} select amount, id, event_time from {{ ref('microbatch_seeds') }} -where {{ incremental_predicate('event_time') }} """ schema_yml = """ diff --git a/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py index 7c25ad99b..8622c1762 100644 --- a/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py +++ b/tests/functional/adapter/microbatch/test_microbatch_insert_by_name.py @@ -2,23 +2,54 @@ from dbt.tests import util from tests.functional.adapter.microbatch.fixtures import ( - microbatch_model_reordered_sql, - microbatch_model_sql, microbatch_seeds_csv, schema_yml, ) +# Uses replace_where strategy (same macro as microbatch) to verify that +# column reordering between runs does not corrupt data. +_initial_model_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='replace_where', + incremental_predicates="id >= 2", +) }} -class TestMicrobatchInsertByName: +{% if not is_incremental() %} +select id, event_time, amount from {{ ref('microbatch_seeds') }} +{% else %} +select id, event_time, amount from {{ ref('microbatch_seeds') }} where id >= 2 +{% endif %} +""" + +# Same logic but columns in a different order (amount, id, event_time) +_reordered_model_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='replace_where', + incremental_predicates="id >= 2", +) }} + +{% if not is_incremental() %} +select amount, id, event_time from {{ ref('microbatch_seeds') }} +{% else %} +select amount, id, event_time from {{ ref('microbatch_seeds') }} where id >= 2 +{% endif %} +""" + + +class TestReplaceWhereColumnOrder: """ - Verifies that the microbatch strategy uses INSERT BY NAME, which preserves - data integrity when column order changes between batches (DBR 12.2+). + Verifies that the replace_where strategy (shared with microbatch) preserves + data integrity when column order changes between the temp and target tables. + + The fix uses INSERT BY NAME so that columns are matched by name, not position. """ @pytest.fixture(scope="class") def models(self): return { - "microbatch_model.sql": microbatch_model_sql, + "replace_where_col_order.sql": _initial_model_sql, "schema.yml": schema_yml, } @@ -28,15 +59,15 @@ def seeds(self): "microbatch_seeds.csv": microbatch_seeds_csv, } - def test_microbatch_insert_by_name(self, project): - # Initial run — creates the table with original column order + def test_replace_where_column_order(self, project): + # Seed and initial run — creates the table with original column order util.run_dbt(["seed"]) util.run_dbt(["run"]) - # Verify initial data using direct SQL comparison and fully qualified name + # Verify initial data actual_initial = project.run_sql( - f"select id, amount from {project.database}.{project.test_schema}.microbatch_model " - "order by id", + f"select id, amount from {project.database}.{project.test_schema}" + ".replace_where_col_order order by id", fetch="all", ) assert len(actual_initial) == 3 @@ -44,32 +75,26 @@ def test_microbatch_insert_by_name(self, project): assert actual_initial[1] == (2, 200) assert actual_initial[2] == (3, 300) - # Simulate column order change by swapping to reordered model - # Add new data to ensure an incremental batch is processed - new_seeds_csv = microbatch_seeds_csv.strip() + "\n4,2023-01-03,400\n" - util.write_file(new_seeds_csv, str(project.project_root) + "/seeds/microbatch_seeds.csv") - + # Swap to the reordered model (amount, id, event_time) util.write_file( - microbatch_model_reordered_sql, - str(project.project_root) + "/models/microbatch_model.sql", + _reordered_model_sql, + str(project.project_root) + "/models/replace_where_col_order.sql", ) - # Second run — should correctly insert by name, not position - util.run_dbt(["seed"]) + # Incremental run with reordered columns util.run_dbt(["run"]) - # Verify row count and data integrity are correct + # Verify data integrity — columns must match by name, not position actual_final = project.run_sql( - f"select id, amount from {project.database}.{project.test_schema}.microbatch_model " - "order by id", + f"select id, amount from {project.database}.{project.test_schema}" + ".replace_where_col_order order by id", fetch="all", ) - assert len(actual_final) == 4 + assert len(actual_final) == 3 + # id=1 was not replaced (predicate is id >= 2), so it stays the same assert actual_final[0] == (1, 100) + # These rows were replaced — if column order was wrong, amount and id + # would be swapped (e.g., id=200 amount=2 instead of id=2 amount=200) assert actual_final[1] == (2, 200) assert actual_final[2] == (3, 300) - - # The critical assertion: amount must be 400, not 4 (which id=4 would produce positionally) - # Without BY NAME, the data from the reordered select would map into the wrong columns. - assert actual_final[3] == (4, 400)