From 37947f10e3d7e0b79b513a6e9015c64754fe945d Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 3 Dec 2024 14:03:17 -0800 Subject: [PATCH 1/6] Preparing for v1.8.4 - Upgraded oracledb driver to 2.5.0 - Dropped support for Python 3.8 --- .github/workflows/oracle-xe-adapter-tests.yml | 2 +- Makefile | 2 +- dbt/adapters/oracle/__version__.py | 2 +- requirements.txt | 2 +- setup.cfg | 7 +++---- setup.py | 9 ++++----- tox.ini | 4 ++-- 7 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/oracle-xe-adapter-tests.yml b/.github/workflows/oracle-xe-adapter-tests.yml index e6544ab..4ab579e 100644 --- a/.github/workflows/oracle-xe-adapter-tests.yml +++ b/.github/workflows/oracle-xe-adapter-tests.yml @@ -8,7 +8,7 @@ jobs: fail-fast: true matrix: os: [ ubuntu-latest ] - python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] services: oracle_db_xe: diff --git a/Makefile b/Makefile index 4cd83df..4a66204 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.8.3 +VERSION=1.8.4 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/adapters/oracle/__version__.py b/dbt/adapters/oracle/__version__.py index 6f8ca09..791f67d 100644 --- a/dbt/adapters/oracle/__version__.py +++ b/dbt/adapters/oracle/__version__.py @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. """ -version = "1.8.3" +version = "1.8.4" diff --git a/requirements.txt b/requirements.txt index c176417..1a17dc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core>=1.8.1,<2.0 -oracledb==2.4.1 +oracledb==2.5.0 diff --git a/setup.cfg b/setup.cfg index 422b8cc..afdc1e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.8.3 +version = 1.8.4 description = dbt (data build tool) adapter for Oracle Autonomous Database long_description = file: README.md long_description_content_type = text/markdown @@ -12,7 +12,6 @@ classifiers = Intended Audience :: Developers License :: OSI Approved :: Apache Software License Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 @@ -28,7 +27,7 @@ project_urls = Release Notes = https://github.com/oracle/dbt-oracle/releases [options] -python_requires = >=3.8 +python_requires = >=3.9 zip_safe = False packages = find_namespace: include_package_data = True @@ -36,7 +35,7 @@ install_requires = dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core~=1.8,<1.9 - oracledb==2.4.1 + oracledb==2.5.0 test_suite=tests test_requires = dbt-tests-adapter~=1.8,<1.9 diff --git a/setup.py b/setup.py index 332cc03..c814923 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ # lockstep with dbt-core which requires Python > 3.8 -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt-oracle does not support this version of Python.") print("Please upgrade to Python 3.8 or higher.") sys.exit(1) @@ -43,7 +43,7 @@ "dbt-common>=1.1.0,<2.0", "dbt-adapters>=1.2.1,<2.0", "dbt-core~=1.8,<1.9", - "oracledb==2.4.1" + "oracledb==2.5.0" ] test_requirements = [ @@ -61,17 +61,16 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION = '1.8.3' +VERSION = '1.8.4' setup( author="Oracle", - python_requires='>=3.8', + python_requires='>=3.9', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', diff --git a/tox.ini b/tox.ini index 8379c06..d2c832a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py3{8,9,10,11,12} +envlist = py3{9,10,11,12} [testenv] passenv = @@ -15,7 +15,7 @@ passenv = deps = -rrequirements.txt - dbt-tests-adapter~=1.7,<1.8 + dbt-tests-adapter~=1.8,<1.9 pytest commands = pytest From d0b2cc316772dd7abe284aa3f3eaa6f4f28e743b Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 3 Dec 2024 14:22:37 -0800 Subject: [PATCH 2/6] Fix https://github.com/oracle/dbt-oracle/issues/158 --- dbt/include/oracle/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/oracle/macros/adapters.sql b/dbt/include/oracle/macros/adapters.sql index 84ab91a..9d254c2 100644 --- a/dbt/include/oracle/macros/adapters.sql +++ b/dbt/include/oracle/macros/adapters.sql @@ -159,9 +159,9 @@ {%- set sql = get_select_subquery(sql) %} {% endif %} {% if temporary -%} on commit preserve rows {%- endif %} + {% if parallel %} parallel {{ parallel }}{% endif %} {% if not temporary -%} {% if partition_clause %} {{ partition_clause }} {% endif %} - {% if parallel %} parallel {{ parallel }}{% endif %} {% if compression_clause %} {{ compression_clause }} {% endif %} {%- endif %} as From 97407c8aaf335ae668ffcfc663a1fdf1547e1fff Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 3 Dec 2024 14:41:14 -0800 Subject: [PATCH 3/6] Fix Snapshots: DATE-field is missing milliseconds, resulting in a gap for the valid period --- .../oracle/macros/materializations/snapshot/snapshot.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql index 9036757..d6f1a9b 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql @@ -164,7 +164,7 @@ {{ strategy.scd_id }} as dbt_scd_id, {{ strategy.updated_at }} as dbt_updated_at, {{ strategy.updated_at }} as dbt_valid_from, - cast(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as DATE) as dbt_valid_to + cast(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as TIMESTAMP) as dbt_valid_to from ( {{ sql }} ) sbq From d0c0b5ba971d00d642aad07fd70a24a4e82d75ff Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 18 Dec 2024 15:36:06 -0800 Subject: [PATCH 4/6] Added a macro to update dbt_scd_id --- .../materializations/snapshot/snapshot.sql | 2 +- .../oracle/macros/update_legacy_snapshots.sql | 70 +++++++++++++++++++ requirements.txt | 2 +- setup.cfg | 2 +- setup.py | 2 +- 5 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 dbt/include/oracle/macros/update_legacy_snapshots.sql diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql index d6f1a9b..6a85fb1 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql @@ -164,7 +164,7 @@ {{ strategy.scd_id }} as dbt_scd_id, {{ strategy.updated_at }} as dbt_updated_at, {{ strategy.updated_at }} as dbt_valid_from, - cast(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as TIMESTAMP) as dbt_valid_to + cast(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as TIMESTAMP(9)) as dbt_valid_to from ( {{ sql }} ) sbq diff --git a/dbt/include/oracle/macros/update_legacy_snapshots.sql b/dbt/include/oracle/macros/update_legacy_snapshots.sql new file mode 100644 index 0000000..a8bd5de --- /dev/null +++ b/dbt/include/oracle/macros/update_legacy_snapshots.sql @@ -0,0 +1,70 @@ +{# + Copyright (c) 2024, Oracle and/or its affiliates. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +{# + Legacy hash function ORA_HASH is known to have collisions in large datasets + causing errors in snapshot merge statement. Please check the below Github + issues: + + https://github.com/oracle/dbt-oracle/issues/52 + https://github.com/oracle/dbt-oracle/issues/102 + +This hash function is used in the marcro oracle__snapshot_hash_arguments + +dbt-oracle 2.0 will switch to a stronger hash function - SHA256. Changing the +hash function will invalidate existing snapshots.These helper macros will +ensure a smoother transition to dbt-oracle 2.0. + +It is recommended for teams to switch to SHA256 hash function before +dbt-oracle 2.0 using a 2-step process: +1. Create a macro oracle__snapshot_hash_arguments(args) in your dbt project + Copy paste the contents of macro + oracle__snapshot_standard_hash_arguments(args) shown below. This will become + the default from dbt-oracle 2.0 + +2. Run the following operation on your snapshot table + +dbt --debug run-operation update_legacy_dbt_scd_id \ + --args '{snapshot_table: PROMOTION_COSTS_SNAPSHOT, cols: ["promo_id", "dbt_updated_at"]}' + +#} + +{% macro oracle__snapshot_standard_hash_arguments(args) -%} + STANDARD_HASH({%- for arg in args -%} + coalesce(cast({{ arg }} as varchar(4000) ), '') + {% if not loop.last %} || '|' || {% endif %} + {%- endfor -%}, 'SHA256') +{%- endmacro %} + + +{% macro update_legacy_dbt_scd_id(snapshot_table, cols) -%} + + {%- call statement('update_legacy_dbt_scd_id_dtype') -%} + BEGIN + UPDATE {{ snapshot_table }} SET DBT_SCD_ID = NULL; + COMMIT; + EXECUTE IMMEDIATE 'ALTER TABLE {{ snapshot_table }} MODIFY (dbt_scd_id RAW(32))'; + END; + {%- endcall -%} + + {%- call statement('update_legacy_dbt_scd_id') -%} + BEGIN + UPDATE {{ snapshot_table }} + SET dbt_scd_id = {{ oracle__snapshot_standard_hash_arguments(cols) }}; + COMMIT; + END; + {%- endcall -%} +{%- endmacro %} diff --git a/requirements.txt b/requirements.txt index 1a17dc5..3b0fccd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core>=1.8.1,<2.0 -oracledb==2.5.0 +oracledb==2.5.1 diff --git a/setup.cfg b/setup.cfg index afdc1e6..bbe8d32 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core~=1.8,<1.9 - oracledb==2.5.0 + oracledb==2.5.1 test_suite=tests test_requires = dbt-tests-adapter~=1.8,<1.9 diff --git a/setup.py b/setup.py index c814923..f4e953b 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ "dbt-common>=1.1.0,<2.0", "dbt-adapters>=1.2.1,<2.0", "dbt-core~=1.8,<1.9", - "oracledb==2.5.0" + "oracledb==2.5.1" ] test_requirements = [ From c8fa8b9b83cb7fd47d8b917252d32efeb9ab3412 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 18 Dec 2024 15:36:41 -0800 Subject: [PATCH 5/6] Updated message to dbt-oracle v1.9.0 --- dbt/include/oracle/macros/update_legacy_snapshots.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/include/oracle/macros/update_legacy_snapshots.sql b/dbt/include/oracle/macros/update_legacy_snapshots.sql index a8bd5de..8d87713 100644 --- a/dbt/include/oracle/macros/update_legacy_snapshots.sql +++ b/dbt/include/oracle/macros/update_legacy_snapshots.sql @@ -24,16 +24,16 @@ This hash function is used in the marcro oracle__snapshot_hash_arguments -dbt-oracle 2.0 will switch to a stronger hash function - SHA256. Changing the +dbt-oracle 1.9 will switch to a stronger hash function - SHA256. Changing the hash function will invalidate existing snapshots.These helper macros will -ensure a smoother transition to dbt-oracle 2.0. +ensure a smoother transition to dbt-oracle 1.9. It is recommended for teams to switch to SHA256 hash function before -dbt-oracle 2.0 using a 2-step process: +dbt-oracle 1.9 using a 2-step process: 1. Create a macro oracle__snapshot_hash_arguments(args) in your dbt project Copy paste the contents of macro oracle__snapshot_standard_hash_arguments(args) shown below. This will become - the default from dbt-oracle 2.0 + the default from dbt-oracle 1.9 2. Run the following operation on your snapshot table From 01f357b95d1da1a3334e012f27b01a67c80ff45a Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 18 Dec 2024 15:39:20 -0800 Subject: [PATCH 6/6] Requires python > 3.9 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4e953b..f8cb5f5 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ # lockstep with dbt-core which requires Python > 3.8 if sys.version_info < (3, 9): print("Error: dbt-oracle does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1)