fivetran · fivetran-joemarkiewicz · Jul 8, 2025 · Jul 2, 2025 · Jul 2, 2025 · Jul 2, 2025
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -31,5 +31,9 @@ dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_abandoned_checkout: false, shopify_using_metafield: false, shopify_using_discount_code_app: true, shopify_product_variant_media: true}' --target "$db" --full-refresh
 dbt test --target "$db"
+if [ "$db" = "bigquery" ]; then
+dbt run --vars '{shopify_collection_identifier: shopify_collection_bq_json_data, shopify_order_identifier: shopify_order_bq_json_data, shopify_transaction_identifier: shopify_transaction_bq_json_data}' --target "$db" --full-refresh
+dbt test --target "$db"
+fi
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 fi
diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
@@ -1,29 +1,35 @@
-## PR Overview
-**This PR will address the following Issue/Feature:**
+<!--
+Pre-Submission Reminders  
+Before marking this PR as "ready for review":
 
-**This PR will result in the following new package version:**
-<!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
+- `dbt run --full-refresh && dbt test`  
+- `dbt run` && `dbt test` (if incremental models are present)  
+- The related issue is linked, tagged, and appropriately assigned  
+- Documentation and version updates are included, if applicable  
+- `docs` have been regenerated (unless there are no code or YAML changes)  
+- BuildKite integration tests are passing
+-->
 
-**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:**
-<!--- Copy/paste the CHANGELOG for this version below. -->
+## PR Overview 
+**Package version introduced in this PR:** 
+
+**This PR addresses the following Issue/Feature(s):**
+<!-- Add Issue # or internal ticket reference -->
 
-## PR Checklist
-### Basic Validation
-Please acknowledge that you have successfully performed the following commands locally:
-- [ ] dbt run –full-refresh && dbt test
-- [ ] dbt run (if incremental models are present) && dbt test
+**Summary of changes:**  
+<!-- 1-2 sentences describing PR changes. -->
 
-Before marking this PR as "ready for review" the following have been applied:
-- [ ] The appropriate issue has been linked, tagged, and properly assigned
-- [ ] All necessary documentation and version upgrades have been applied
-- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
-- [ ] BuildKite integration tests are passing
-- [ ] Detailed validation steps have been provided below
 
-### Detailed Validation
-Please share any and all of your validation steps:
-<!--- Provide the steps you took to validate your changes below. -->
+### Submission Checklist  
+- [ ] Alignment meeting with the reviewer (if needed)  
+  - [ ] Timeline and validation requirements discussed  
+- [ ] Provide validation details:  
+  - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests)
+  - [ ] **Testing Instructions:** Confirm the change addresses the issue(s)
+  - [ ] **Focus Areas:** Complex logic or queries that need extra attention  
 
-### If you had to summarize this PR in an emoji, which would it be?
-<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
-:dancer:
+### Changelog  
+<!-- Recommend drafting changelog notes, then refining via ChatGPT using:  
+"Draft a changelog entry based on the following notes." -->
+- [ ] Draft changelog for PR  
+- [ ] Final changelog for release review
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+# dbt_shopify_source v0.18.1
+
+[PR #103](https://github.com/fivetran/dbt_shopify_source/pull/103) includes the following updates:
+
+## Under the Hood
+- Added BigQuery JSON field support for the following model and columns:
+  - `stg_shopify__order`: `total_shipping_price_set`, `total_discounts_set`, `total_line_items_price_set`, `total_price_set`, and `total_tax_set` columns
+  - `stg_shopify__transaction`: `receipt` column
+  - `stg_shopify__collection`: `rules` column
+  - `stg_shopify_shop`: `enabled_presentment_currencies` column
+- Added the `json_to_string()` macro for BigQuery to convert JSON fields to strings for reliable downstream parsing.
+- Included json versions to the integration tests to ensure json data type compatibility.
+- Updated the Maintainer PR template.
+
 # dbt_shopify_source v0.18.0
 
 [PR #100](https://github.com/fivetran/dbt_shopify_source/pull/100) includes the following updates:

diff --git a/LICENSE b/LICENSE
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright © 2025 Fivetran Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_source'
-version: '0.18.0'
+version: '0.18.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_source_integration_tests'
-version: '0.18.0'
+version: '0.18.1'
 profile: 'integration_tests'
 config-version: 2
 
@@ -85,6 +85,21 @@ seeds:
         customer_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         location_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         user_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        total_shipping_price_set: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+    shopify_order_bq_json_data:
+      +enabled: "{{ target.type == 'bigquery' }}"
+      +column_types:
+        created_at: timestamp
+        updated_at: timestamp
+        processed_at: timestamp
+        cancelled_at: timestamp
+        _fivetran_synced: timestamp
+        total_tax: float
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        customer_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        location_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        user_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        total_shipping_price_set: json
     shopify_order_line_data:
       +column_types:
         _fivetran_synced: timestamp
@@ -126,6 +141,14 @@ seeds:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         refund_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        receipt: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+    shopify_transaction_bq_json_data:
+      +enabled: "{{ target.type == 'bigquery' }}"
+      +column_types:
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        refund_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        receipt: json
     shopify_refund_data:
       +column_types:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
@@ -169,6 +192,14 @@ seeds:
         disjunctive: boolean
         published_at: timestamp
         updated_at: timestamp
+        rules: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+    shopify_collection_bq_json_data:
+      +enabled: "{{ target.type == 'bigquery' }}"
+      +column_types:
+        disjunctive: boolean
+        published_at: timestamp
+        updated_at: timestamp
+        rules: json
     shopify_metafield_data:
       +column_types:
         value_type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"

diff --git a/integration_tests/seeds/shopify_collection_bq_json_data.csv b/integration_tests/seeds/shopify_collection_bq_json_data.csv
@@ -0,0 +1,4 @@
+id,_fivetran_deleted,_fivetran_synced,handle,published_at,published_scope,title,updated_at,disjunctive,rules,sort_order,template_suffix,body_html
+997355,true,2021-09-01 05:53:25.838000,,,,title_1,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""TEST""}]",,,
+9930779,true,2021-09-01 05:53:26.673000,,,,title_2,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""FOO""}]",,,
+99967,true,2022-04-08 06:52:19.524000,,,,title_3,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""BAR""}]",,,
diff --git a/integration_tests/seeds/shopify_collection_data.csv b/integration_tests/seeds/shopify_collection_data.csv
@@ -1,4 +1,4 @@
 id,_fivetran_deleted,_fivetran_synced,handle,published_at,published_scope,title,updated_at,disjunctive,rules,sort_order,template_suffix,body_html
-997355,true,2021-09-01 05:53:25.838000,,,,title_1,1970-01-01 00:00:00.000000,,,,,
-9930779,true,2021-09-01 05:53:26.673000,,,,title_2,1970-01-01 00:00:00.000000,,,,,
-99967,true,2022-04-08 06:52:19.524000,,,,title_3,1970-01-01 00:00:00.000000,,,,,
+997355,true,2021-09-01 05:53:25.838000,,,,title_1,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""TEST""}]",,,
+9930779,true,2021-09-01 05:53:26.673000,,,,title_2,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""FOO""}]",,,
+99967,true,2022-04-08 06:52:19.524000,,,,title_3,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""BAR""}]",,,
diff --git a/integration_tests/seeds/shopify_order_bq_json_data.csv b/integration_tests/seeds/shopify_order_bq_json_data.csv
@@ -0,0 +1,9 @@
+id,note,email,taxes_included,currency,subtotal_price,total_tax,total_price,created_at,updated_at,name,shipping_address_name,shipping_address_first_name,shipping_address_last_name,shipping_address_company,shipping_address_phone,shipping_address_address_1,shipping_address_address_2,shipping_address_city,shipping_address_country,shipping_address_country_code,shipping_address_province,shipping_address_province_code,shipping_address_zip,shipping_address_latitude,shipping_address_longitude,billing_address_name,billing_address_first_name,billing_address_last_name,billing_address_company,billing_address_phone,billing_address_address_1,billing_address_address_2,billing_address_city,billing_address_country,billing_address_country_code,billing_address_province,billing_address_province_code,billing_address_zip,billing_address_latitude,billing_address_longitude,customer_id,location_id,user_id,number,order_number,financial_status,fulfillment_status,processed_at,processing_method,referring_site,cancel_reason,cancelled_at,closed_at,total_discounts,total_line_items_price,total_weight,source_name,browser_ip,buyer_accepts_marketing,token,cart_token,checkout_token,test,landing_site_base_url,_fivetran_synced,total_shipping_price_set
+2674098602081,71509c29301d2cc14e37ecb53f735608,021cb20b5c78751fc7ddc091b6b69b3e,TRUE,GBP,2.8,0.00,2.8,2020-09-11 19:35:42.000,2020-09-11 19:35:46.000,d1743fc58a1e4d78769eaac49994a994,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,3589760876641,,,4135,5135,paid,,2020-09-11 19:35:42.000,,,,,,2.8,5.6,0,app,,TRUE,0f9c2880de17f71511eee5542c29b999,,,FALSE,,2020-09-12 00:15:10.199 ,"{""shop_money"":{""amount"":""3.00"",""currency_code"":""USD""}}"
+2669516488801,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,2.8,0.00,3.79,2020-09-09 23:01:54.000,2020-09-10 15:38:26.000,4fcb884b5b46413bae526a6e7e49d706,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4066,5066,paid,fulfilled,2020-09-09 23:01:53.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:26.000,0,2.8,0,web,109.249.185.68,FALSE,fb489b3ccc0ae36ce47744d7595e9746,b1ff04883dfeab658cd5211050476729,7bdb994e1196de3e4f34586e357613f9,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.536 ,"{""shop_money"":{""amount"":""4.00"",""currency_code"":""USD""}}"
+2669509541985,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,4.4,0.00,5.39,2020-09-09 22:57:51.000,2020-09-10 15:38:25.000,9e346f2e912c60e16679f4a4c8d29422,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4065,5065,paid,fulfilled,2020-09-09 22:57:50.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:25.000,0,4.4,0,web,109.249.185.68,FALSE,e44b7f04610a8f4032530cc7f12663de,9600543f4d4613db59ac58a1009ecbb9,cf0a9fe2c7c606b86559007dbb890a62,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.037 ,"{""shop_money"":{""amount"":""5.00"",""currency_code"":""USD""}}"
+1000001,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,100.00,0.01,100.01,2025-05-01 08:00:00.000,2025-05-01 08:10:00.000,9e346f2e912c60e16679f4a4c8d29423,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10001,20001,30001,1001,1001,paid,,2025-05-01 08:00:00.000,direct,,,,,15.00,100.00,500,web,192.168.1.1,FALSE,t0ken1001,cart1001,checkout1001,FALSE,,2025-05-01 08:15:00.000,
+1000002,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,80.00,0.02,100.02,2025-05-02 09:00:00.000,2025-05-02 09:10:00.000,9e346f2e912c60e16679f4a4c8d29424,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10002,20002,30002,1002,1002,paid,,2025-05-02 09:00:00.000,direct,,,,,20.00,80.00,450,web,192.168.1.2,FALSE,t0ken1002,cart1002,checkout1002,FALSE,,2025-05-02 09:15:00.000,"{""shop_money"":{""amount"":""6.00"",""currency_code"":""USD""}}"
+1000003,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,50.00,0.00,50.00,2025-05-03 10:15:00.000,2025-05-03 10:20:00.000,9e346f2e912c60e16679f4a4c8d29425,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10003,20003,30003,1003,1003,paid,,2025-05-03 10:15:00.000,direct,,,,,0.00,50.00,250,web,192.168.1.3,FALSE,t0ken1003,cart1003,checkout1003,FALSE,,2025-05-03 10:30:00.000,
+1000004,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,200.00,0.00,205.00,2025-05-04 11:30:00.000,2025-05-04 11:35:00.000,9e346f2e912c60e16679f4a4c8d29426,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10004,20004,30004,1004,1004,paid,,2025-05-04 11:30:00.000,direct,,,,,5.00,200.00,750,web,192.168.1.4,FALSE,t0ken1004,cart1004,checkout1004,FALSE,,2025-05-04 11:45:00.000,"{""shop_money"":{""amount"":""7.00"",""currency_code"":""USD""}}"
+1000005,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,150.00,0.00,160.00,2025-05-05 14:00:00.000,2025-05-05 14:10:00.000,9e346f2e912c60e16679f4a4c8d29427,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10005,20005,30005,1005,1005,paid,,2025-05-05 14:00:00.000,direct,,,,,10.00,150.00,600,web,192.168.1.5,FALSE,t0ken1005,cart1005,checkout1005,FALSE,,2025-05-05 14:20:00.000,"{""shop_money"":{""amount"":""8.00"",""currency_code"":""USD""}}"