Merge pull request #10 from oracle/v1.0.1

Release v1.0.1

Author: aosingh

.github/scripts/install_oracle_instantclient.sh

@@ -0,0 +1,10 @@
+sudo apt-get update
+sudo apt-get install wget libaio1
+sudo mkdir -p /opt/oracle
+wget https://download.oracle.com/otn_software/linux/instantclient/216000/instantclient-basiclite-linux.x64-21.6.0.0.0dbru.zip -P /tmp
+sudo unzip /tmp/instantclient-basiclite-linux.x64-21.6.0.0.0dbru.zip -d /opt/oracle
+export PATH="$PATH:/opt/oracle/instantclient_21_6"
+export LD_LIBRARY_PATH="/opt/oracle/instantclient_21_6:$LD_LIBRARY_PATH"
+sudo mkdir -p /opt/tns_admin
+echo "DISABLE_OOB=ON" >> /opt/tns_admin/sqlnet.ora
+

.github/workflows/oracle-xe-adapter-tests.yml

@@ -0,0 +1,112 @@
+name: dbt-adapter-tests
+on: push
+
+jobs:
+  oracle-21_3_0-xe-test-suite:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ ubuntu-20.04 ]
+        python-version: [ 3.7, 3.8, 3.9 ]
+
+    services:
+      oracle_db_xe:
+        image: container-registry.oracle.com/database/express:21.3.0-xe
+        env:
+          ORACLE_PWD: ${{ secrets.DBT_ORACLE_PASSWORD }}
+        ports:
+          - 1521:1521
+
+    steps:
+      - name: Check out dbt-oracle repository code
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Oracle Instantclient
+        run: |
+          chmod +x ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
+          ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
+
+      - name: Install dbt-oracle with core dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest pytest-dbt-adapter==0.6.0
+          pip install -r requirements.txt
+          pip install -e .
+
+      - name: Run adapter tests
+        run: |
+          pytest tests/oracle.dbtspec tests/test_config.py
+        env:
+          DBT_ORACLE_USER: ${{ secrets.DBT_ORACLE_USER }}
+          DBT_ORACLE_HOST: localhost
+          DBT_ORACLE_PORT: 1521
+          DBT_ORACLE_SCHEMA: ${{ secrets.DBT_ORACLE_USER }}
+          DBT_ORACLE_PASSWORD: ${{ secrets.DBT_ORACLE_PASSWORD }}
+          DBT_ORACLE_DATABASE: XEPDB1
+          DBT_ORACLE_SERVICE: XEPDB1
+          DBT_ORACLE_PROTOCOL: tcp
+          LD_LIBRARY_PATH: /opt/oracle/instantclient_21_6
+          TNS_ADMIN: /opt/tns_admin
+
+  oracle-21_3_0-xe-py36-test-suite:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ ubuntu-20.04 ]
+        python-version: [ 3.6 ]
+
+    services:
+      oracle_db_xe:
+        image: container-registry.oracle.com/database/express:21.3.0-xe
+        env:
+          ORACLE_PWD: ${{ secrets.DBT_ORACLE_PASSWORD }}
+        ports:
+          - 1521:1521
+
+    steps:
+      - name: Check out dbt-oracle repository code
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Oracle Instantclient
+        run: |
+          chmod +x ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
+          ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
+
+      - name: Install dbt-oracle with core dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest pytest-dbt-adapter==0.4.0
+          pip install -r requirements.txt
+          pip install -e .
+
+      - name: Run adapter tests
+        run: |
+          pytest tests/oracle_py36.dbtspec tests/test_config.py
+        env:
+          DBT_ORACLE_USER: ${{ secrets.DBT_ORACLE_USER }}
+          DBT_ORACLE_HOST: localhost
+          DBT_ORACLE_PORT: 1521
+          DBT_ORACLE_SCHEMA: ${{ secrets.DBT_ORACLE_USER }}
+          DBT_ORACLE_PASSWORD: ${{ secrets.DBT_ORACLE_PASSWORD }}
+          DBT_ORACLE_DATABASE: XEPDB1
+          DBT_ORACLE_SERVICE: XEPDB1
+          DBT_ORACLE_PROTOCOL: tcp
+          LD_LIBRARY_PATH: /opt/oracle/instantclient_21_6
+          TNS_ADMIN: /opt/tns_admin
+
+
+
+
+

README.md

@@ -1,5 +1,8 @@
 # dbt-oracle
 
+[![PyPI version](https://badge.fury.io/py/dbt-oracle.svg)](https://pypi.python.org/pypi/dbt-oracle)
+![Build](https://github.com/oracle/dbt-oracle/actions/workflows/oracle-xe-adapter-tests.yml/badge.svg)
+
 dbt "adapters" are responsible for adapting dbt's functionality to a given database. `dbt-oracle` implements dbt functionalities for the Oracle database. To learn more about building adapters, check
 https://docs.getdbt.com/docs/contributing/building-a-new-adapter
 

dbt/adapters/oracle/connections.py

@@ -14,7 +14,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-from typing import List, Optional, Tuple, Any
+from typing import List, Optional, Tuple, Any, Dict
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 import enum
@@ -56,7 +56,8 @@ class OracleAdapterCredentials(Credentials):
     # Mandatory required arguments.
     user: str
     password: str
-    database: str
+    # Specifying database is optional
+    database: Optional[str]
 
     # OracleConnectionMethod.TNS
     tns_name: Optional[str] = None
@@ -104,6 +105,14 @@ def _connection_keys(self) -> Tuple[str]:
             'cclass', 'purity'
         )
 
+    @classmethod
+    def __pre_deserialize__(cls, data: Dict[Any, Any]) -> Dict[Any, Any]:
+        # If database is not defined as adapter credentials
+        data = super().__pre_deserialize__(data)
+        if "database" not in data:
+            data["database"] = None
+        return data
+
     def connection_method(self) -> OracleConnectionMethod:
         """Return an OracleConnecitonMethod inferred from the configuration"""
         if self.connection_string:
@@ -124,16 +133,8 @@ def get_dsn(self) -> str:
         if method == OracleConnectionMethod.CONNECTION_STRING:
             return self.connection_string
 
-        # Assume host connection method OracleConnectionMethod.HOST
-
-        # If the 'service' property is not provided, use 'database' property for
-        # purposes of connecting.
-        if self.service:
-            service = self.service
-        else:
-            service = self.database
-
-        return f'{self.protocol}://{self.host}:{self.port}/{service}'
+        # Assume host connection method OracleConnectionMethod.HOST and necessary parameters are defined
+        return f'{self.protocol}://{self.host}:{self.port}/{self.service}'
 
 
 class OracleAdapterConnectionManager(SQLConnectionManager):

dbt/adapters/oracle/impl.py

@@ -15,14 +15,19 @@
   limitations under the License.
 """
 from typing import (
-    Optional, List
+    Optional, List, Set
 )
+from itertools import chain
 
 import dbt.exceptions
+from dbt.adapters.base.relation import BaseRelation, InformationSchema
+from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME
 from dbt.adapters.sql import SQLAdapter
 from dbt.adapters.base.meta import available
 from dbt.adapters.oracle import OracleAdapterConnectionManager
 from dbt.adapters.oracle.relation import OracleRelation
+from dbt.contracts.graph.manifest import Manifest
+
 
 
 import agate
@@ -56,6 +61,7 @@
 '''.strip()
 
 LIST_RELATIONS_MACRO_NAME = 'list_relations_without_caching'
+GET_DATABASE_MACRO_NAME = 'get_database_name'
 
 
 class OracleAdapter(SQLAdapter):
@@ -103,7 +109,7 @@ def verify_database(self, database):
         if database.startswith('"'):
             database = database.strip('"')
         expected = self.config.credentials.database
-        if database.lower() != expected.lower():
+        if expected and database.lower() != expected.lower():
             raise dbt.exceptions.NotImplementedException(
                 'Cross-db references not allowed in {} ({} vs {})'
                 .format(self.type(), database, expected)
@@ -150,3 +156,59 @@ def timestamp_add_sql(
         # '+ interval' syntax used in postgres/redshift is relatively common
         # and might even be the SQL standard's intention.
         return f"{add_to} + interval '{number}' {interval}"
+
+    def get_relation(self, database: str, schema: str, identifier: str) -> Optional[BaseRelation]:
+        if database == 'None':
+            database = self.config.credentials.database
+        return super().get_relation(database, schema, identifier)
+
+    def _get_one_catalog(
+            self,
+            information_schema: InformationSchema,
+            schemas: Set[str],
+            manifest: Manifest,
+    ) -> agate.Table:
+
+        kwargs = {"information_schema": information_schema, "schemas": schemas}
+        table = self.execute_macro(
+            GET_CATALOG_MACRO_NAME,
+            kwargs=kwargs,
+            # pass in the full manifest so we get any local project
+            # overrides
+            manifest=manifest,
+        )
+        # In case database is not defined, we can use the the configured database which we set as part of credentials
+        for node in chain(manifest.nodes.values(), manifest.sources.values()):
+            if not node.database or node.database == 'None':
+                node.database = self.config.credentials.database
+
+        results = self._catalog_filter_table(table, manifest)
+        return results
+
+    def list_relations_without_caching(
+            self, schema_relation: BaseRelation,
+    ) -> List[BaseRelation]:
+
+        # Set database if not supplied
+        if not self.config.credentials.database:
+            self.config.credentials.database = self.execute_macro(GET_DATABASE_MACRO_NAME)
+
+        kwargs = {'schema_relation': schema_relation}
+        results = self.execute_macro(
+            LIST_RELATIONS_MACRO_NAME,
+            kwargs=kwargs
+        )
+        relations = []
+        for _database, name, _schema, _type in results:
+            try:
+                _type = self.Relation.get_relation_type(_type)
+            except ValueError:
+                _type = self.Relation.External
+            relations.append(self.Relation.create(
+                database=_database,
+                schema=_schema,
+                identifier=name,
+                quote_policy=self.config.quoting,
+                type=_type
+            ))
+        return relations

dbt/adapters/oracle/sample_profiles.yml

@@ -9,12 +9,5 @@ default:
          password: "{{ env_var('DBT_ORACLE_PASSWORD') }}"
          port: 1522
          service: "{{ env_var('DBT_ORACLE_SERVICE') }}"
-         dbname: "{{ env_var('DBT_ORACLE_DATABASE') }}"
          schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}"
          threads: 1
-         shardingkey:
-            - skey
-         supershardingkey:
-            - sskey
-         cclass: CONNECTIVITY_CLASS
-         purity: self|new|default

dbt/include/oracle/macros/adapters.sql

@@ -182,9 +182,6 @@
         {% if relation.schema %}
         and table_schema = upper('{{ relation.schema }}')
         {% endif %}
-        {% if relation.database %}
-        and table_catalog = upper('{{ relation.database }}')
-        {% endif %}
       order by ordinal_position
 
   {% endcall %}
@@ -341,7 +338,6 @@
     end as "kind"
   from tables
   where table_type in ('BASE TABLE', 'VIEW')
-    and table_catalog = upper('{{ schema_relation.database }}')
     and table_schema = upper('{{ schema_relation.schema }}')
   {% endcall %}
   {{ return(load_result('list_relations_without_caching').table) }}
@@ -360,3 +356,9 @@
 
     {% do return(tmp_relation) %}
 {% endmacro %}
+
+{% macro get_database_name() %}
+    {% set results = run_query("select SYS_CONTEXT('userenv', 'DB_NAME') FROM DUAL") %}
+    {% set db_name = results.columns[0].values()[0] %}
+    {{ return(db_name) }}
+{% endmacro %}

dbt/include/oracle/macros/catalog.sql

@@ -22,6 +22,9 @@
       But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.
     #}
     {% set database = information_schema.database %}
+    {% if database == 'None' or database is undefined or database is none %}
+      {% set database = get_database_name() %}
+    {% endif %}
     {{ adapter.verify_database(database) }}
 
     with columns as (

dbt/include/oracle/macros/materializations/snapshot/snapshot.sql

@@ -208,12 +208,17 @@
   {%- set strategy_name = config.get('strategy') -%}
   {%- set unique_key = config.get('unique_key') %}
 
-  {% if not adapter.check_schema_exists(model.database, model.schema) %}
-    {% do create_schema(model.database, model.schema) %}
+  {% set model_database = model.database %}
+  {% if model_database == 'None' or model_database is undefined or model_database is none %}
+    {% set model_database = get_database_name() %}
+  {% endif %}
+
+  {% if not adapter.check_schema_exists(model_database, model.schema) %}
+    {% do create_schema(model_database, model.schema) %}
   {% endif %}
 
   {% set target_relation_exists, target_relation = get_or_create_relation(
-          database=model.database,
+          database=model_database,
           schema=model.schema,
           identifier=target_table,
           type='table') -%}

dbt/include/oracle/macros/materializations/snapshot/strategies.sql

@@ -77,8 +77,13 @@
         {{ return([false, query_columns]) }}
     {%- endif -%}
     {# handle any schema changes #}
+    {% set node_database = node.database %}
+    {% if node_database == 'None' or model_database is undefined or model_database is none %}
+        {% set node_database = get_database_name() %}
+    {% endif %}
     {%- set target_table = node.get('alias', node.get('name')) -%}
-    {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}
+
+    {%- set target_relation = adapter.get_relation(database=node_database, schema=node.schema, identifier=target_table) -%}
     {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation.quote(schema=False, identifier=False)) -%}
     {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}
     {%- set ns.column_added = false -%}

dbt/include/oracle/profile_template.yml

@@ -21,10 +21,6 @@ prompts:
     hint: 'service name in tnsnames.ora'
     type: string
     default: "{{ env_var('DBT_ORACLE_SERVICE') }}"
-  dbname:
-    hint: 'database name in which dbt objects should be created'
-    type: string
-    default: "{{ env_var('DBT_ORACLE_DATABASE') }}"
   schema:
     hint: 'database schema in which dbt objects should be created'
     type: string

dbt_adbs_test_project/profiles.yml

@@ -9,7 +9,7 @@ dbt_test:
          host: "{{ env_var('DBT_ORACLE_HOST') }}"
          port: 1522
          service: "{{ env_var('DBT_ORACLE_SERVICE') }}"
-         database: "{{ env_var('DBT_ORACLE_DATABASE') }}"
+         #database: "{{ env_var('DBT_ORACLE_DATABASE') }}"
          schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}"
          shardingkey:
            - skey

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dbt-oracle
-version = 1.0.0
+version = 1.0.1
 description = dbt (data build tool) adapter for the Oracle database
 long_description = file: README.md
 long_description_content_type = text/markdown