From d2e6682f35932301d5a55cc4207e56d56cb04f77 Mon Sep 17 00:00:00 2001 From: kylemcnair <40182913+kylemcnair@users.noreply.github.com> Date: Tue, 21 Mar 2023 17:40:27 -0700 Subject: [PATCH 1/3] dev events --- data_diff/tracking.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/data_diff/tracking.py b/data_diff/tracking.py index 582ebf24..53f0b98b 100644 --- a/data_diff/tracking.py +++ b/data_diff/tracking.py @@ -78,6 +78,7 @@ def create_start_event_json(diff_options: Dict[str, Any]): "diff_options": diff_options, "data_diff_version:": __version__, "entrypoint_name": entrypoint_name, + "dbt_user_id": "1234" }, } @@ -112,6 +113,7 @@ def create_end_event_json( "entrypoint_name": entrypoint_name, "is_cloud": is_cloud, "diff_id": diff_id, + "dbt_user_id": "456" }, } @@ -122,7 +124,8 @@ def send_event_json(event_json): headers = { "Content-Type": "application/json", - "Authorization": "Basic MkhndE00SGNxOUJtZWlDcU5ZaHo3Tzl0a2pNOg==", + #"Authorization": "Basic MkhndE00SGNxOUJtZWlDcU5ZaHo3Tzl0a2pNOg==", #prod + "Authorization": "Basic MkhnZ2o5NWhOc0JzeFNBNW1IaXJQYkk5d3pXOg==", } data = json.dumps(event_json).encode() try: From c963fa134c3053577e2362f767493d37679b915d Mon Sep 17 00:00:00 2001 From: kylemcnair <40182913+kylemcnair@users.noreply.github.com> Date: Wed, 22 Mar 2023 11:59:28 -0700 Subject: [PATCH 2/3] wip --- data_diff/dbt.py | 11 +++++++++++ data_diff/tracking.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/data_diff/dbt.py b/data_diff/dbt.py index 0baec9d5..d3398a73 100644 --- a/data_diff/dbt.py +++ b/data_diff/dbt.py @@ -80,6 +80,7 @@ def dbt_diff( custom_schemas = datadiff_variables.get("custom_schemas") # custom schemas is default dbt behavior, so default to True if the var doesn't exist custom_schemas = True if custom_schemas is None else custom_schemas + dbt_user_id = dbt_parser.get_dbt_user_id("dbt_user_id") if not is_cloud: dbt_parser.set_connection() @@ -278,6 +279,7 @@ def _cloud_diff(diff_vars: DiffVars) -> None: error=err_message, diff_id=diff_id, is_cloud=True, + dbt_user_id="", ) send_event_json(event_json) @@ -327,6 +329,15 @@ def get_models(self): rich.print(f"Found {str(len(models))} successful model runs from the last dbt command.") return models + def get_dbt_user_id(self): + with open(DbtParser.project_dir / MANIFEST_PATH) as manifest: + manifest_dict = json.load(manifest) + manifest_obj = self.parse_manifest(manifest=manifest_dict) + + dbt_user_id = parse_version(manifest_obj.metadata.user_id) + + return dbt_user_id + def get_primary_keys(self, model): return list((x.name for x in model.columns.values() if "primary-key" in x.tags)) diff --git a/data_diff/tracking.py b/data_diff/tracking.py index 53f0b98b..f56b8a12 100644 --- a/data_diff/tracking.py +++ b/data_diff/tracking.py @@ -94,6 +94,7 @@ def create_end_event_json( error: Optional[str], diff_id: Optional[int] = None, is_cloud: bool = False, + #dbt_user_id: str, ): return { "event": "os_diff_run_end", @@ -113,7 +114,7 @@ def create_end_event_json( "entrypoint_name": entrypoint_name, "is_cloud": is_cloud, "diff_id": diff_id, - "dbt_user_id": "456" + "dbt_user_id": dbt_user_id, }, } From 3be090a60c87b891a5dc0a98eb385533db4fd2ad Mon Sep 17 00:00:00 2001 From: kylemcnair <40182913+kylemcnair@users.noreply.github.com> Date: Wed, 22 Mar 2023 12:03:39 -0700 Subject: [PATCH 3/3] wip --- data_diff/dbt.py | 1 - data_diff/tracking.py | 1 - 2 files changed, 2 deletions(-) diff --git a/data_diff/dbt.py b/data_diff/dbt.py index d3398a73..1b515cc4 100644 --- a/data_diff/dbt.py +++ b/data_diff/dbt.py @@ -279,7 +279,6 @@ def _cloud_diff(diff_vars: DiffVars) -> None: error=err_message, diff_id=diff_id, is_cloud=True, - dbt_user_id="", ) send_event_json(event_json) diff --git a/data_diff/tracking.py b/data_diff/tracking.py index f56b8a12..d1a8d854 100644 --- a/data_diff/tracking.py +++ b/data_diff/tracking.py @@ -78,7 +78,6 @@ def create_start_event_json(diff_options: Dict[str, Any]): "diff_options": diff_options, "data_diff_version:": __version__, "entrypoint_name": entrypoint_name, - "dbt_user_id": "1234" }, }