Merge branch 'master' into master

Author: kindly

README.md

@@ -77,6 +77,8 @@ Test SQL code and preview changes by comparing development/staging environment d
 
 **[Get started with data-diff & dbt](https://docs.datafold.com/development_testing/open_source)**
 
+Also available in a [VS Code Extension](https://marketplace.visualstudio.com/items?itemName=Datafold.datafold-vscode)
+
 Reach out on the dbt Slack in [#tools-datafold](https://getdbt.slack.com/archives/C03D25A92UU) for advice and support
 
 # Supported databases

data_diff/__main__.py

@@ -48,8 +48,8 @@ def _get_log_handlers(is_dbt: Optional[bool] = False) -> Dict[str, logging.Handl
     rich_handler.setLevel(logging.WARN)
     handlers["rich_handler"] = rich_handler
 
-    # only use log_status_handler in a terminal
-    if rich_handler.console.is_terminal and is_dbt:
+    # only use log_status_handler in an interactive terminal session
+    if rich_handler.console.is_interactive and is_dbt:
         log_status_handler = LogStatusHandler()
         log_status_handler.setFormatter(logging.Formatter(log_format_status, datefmt=date_format))
         log_status_handler.setLevel(logging.DEBUG)

data_diff/config.py

@@ -4,6 +4,12 @@
 import toml
 
 
+_ARRAY_FIELDS = (
+    "key_columns",
+    "columns",
+)
+
+
 class ConfigParseError(Exception):
     pass
 
@@ -38,6 +44,11 @@ def _apply_config(config: Dict[str, Any], run_name: str, kw: Dict[str, Any]):
         for index in "12":
             run_args[index] = {attr: kw.pop(f"{attr}{index}") for attr in ("database", "table")}
 
+    # Make sure array fields are decoded as list, since array fields in toml are decoded as list, but TableSegment object requires tuple type.
+    for field in _ARRAY_FIELDS:
+        if isinstance(run_args.get(field), list):
+            run_args[field] = tuple(run_args[field])
+
     # Process databases + tables
     for index in "12":
         try:

data_diff/dbt.py

@@ -23,6 +23,7 @@
 from .format import jsonify, jsonify_error
 from .tracking import (
     bool_ask_for_email,
+    bool_notify_about_extension,
     create_email_signup_event_json,
     set_entrypoint_name,
     set_dbt_user_id,
@@ -48,6 +49,7 @@
 
 logger = getLogger(__name__)
 CLOUD_DOC_URL = "https://docs.datafold.com/development_testing/cloud"
+EXTENSION_INSTALL_URL = "https://get.datafold.com/datafold-vs-code-install"
 
 
 class TDiffVars(pydantic.BaseModel):
@@ -155,6 +157,8 @@ def dbt_diff(
             for thread in diff_threads:
                 thread.join()
 
+    _extension_notification()
+
 
 def _get_diff_vars(
     dbt_parser: "DbtParser",
@@ -517,3 +521,10 @@ def _email_signup() -> None:
         if email:
             event_json = create_email_signup_event_json(email)
             run_as_daemon(send_event_json, event_json)
+
+
+def _extension_notification() -> None:
+    if bool_notify_about_extension():
+        rich.print(
+            f"\n\nHaving a good time diffing? :heart_eyes-emoji:\nMake sure to check out the free [bold]Datafold VS Code extension[/bold] for more a more seamless diff experience:\n{EXTENSION_INSTALL_URL}"
+        )

data_diff/tracking.py

@@ -11,6 +11,7 @@
 import urllib.request
 from uuid import uuid4
 import toml
+from rich import get_console
 
 from .version import __version__
 
@@ -41,7 +42,7 @@ def bool_ask_for_email() -> bool:
     """
     Checks the .datadiff.toml profile file for the asked_for_email key
 
-    Returns False immediately if --no-tracking
+    Returns False immediately if --no-tracking or not in an interactive terminal
 
     If found, return False (already asked for email)
 
@@ -50,7 +51,8 @@ def bool_ask_for_email() -> bool:
     Returns:
         bool: decision on whether to prompt the user for their email
     """
-    if g_tracking_enabled:
+    console = get_console()
+    if g_tracking_enabled and console.is_interactive:
         profile = _load_profile()
 
         if "asked_for_email" not in profile:
@@ -61,6 +63,17 @@ def bool_ask_for_email() -> bool:
     return False
 
 
+def bool_notify_about_extension() -> bool:
+    profile = _load_profile()
+    console = get_console()
+    if "notified_about_extension" not in profile and console.is_interactive:
+        profile["notified_about_extension"] = ""
+        with open(DEFAULT_PROFILE, "w") as conf:
+            toml.dump(profile, conf)
+        return True
+    return False
+
+
 g_tracking_enabled = True
 g_anonymous_id = None
 

data_diff/version.py

@@ -1 +1 @@
-__version__ = "0.8.1"
+__version__ = "0.8.2"

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-diff"
-version = "0.8.1"
+version = "0.8.2"
 description = "Command-line tool and Python library to efficiently diff rows across two different databases."
 authors = ["Datafold <data-diff@datafold.com>"]
 license = "MIT"
@@ -36,7 +36,7 @@ cryptography = {version="*", optional=true}
 trino = {version="^0.314.0", optional=true}
 presto-python-client = {version="*", optional=true}
 clickhouse-driver = {version="*", optional=true}
-duckdb = {version="^0.7.0", optional=true}
+duckdb = {version="*", optional=true}
 dbt-artifacts-parser = {version="^0.4.0"}
 dbt-core = {version="^1.0.0"}
 keyring = "*"

pyproject.toml

@@ -15,6 +15,8 @@ def test_basic(self):
 
             [run.default]
             update_column = "timestamp"
+            key_columns = ["id"]
+            columns = ["name", "age"]
             verbose = true
             threads = 2
 
@@ -39,6 +41,8 @@ def test_basic(self):
         assert res["table2"] == "rating_del1"
         assert res["threads1"] == 11
         assert res["threads2"] == 22
+        assert res["key_columns"] == ("id",)
+        assert res["columns"] == ("name", "age")
 
         res = apply_config_from_string(config, "pg_pg", {"update_column": "foo", "table2": "bar"})
         assert res["update_column"] == "foo"