diff --git a/data_diff/cloud/datafold_api.py b/data_diff/cloud/datafold_api.py index 7a28ef97..b6c4531b 100644 --- a/data_diff/cloud/datafold_api.py +++ b/data_diff/cloud/datafold_api.py @@ -7,7 +7,7 @@ import pydantic import requests -from data_diff.errors import DataDiffDatasourceIdNotFoundError +from data_diff.errors import DataDiffCloudDiffFailed, DataDiffCloudDiffTimedOut, DataDiffDatasourceIdNotFoundError from ..utils import getLogger @@ -248,8 +248,8 @@ def create_data_diff(self, payload: TCloudApiDataDiff) -> int: def poll_data_diff_results(self, diff_id: int) -> TCloudApiDataDiffSummaryResult: summary_results = None start_time = time.monotonic() - sleep_interval = 5 # starts at 5 sec - max_sleep_interval = 30 + sleep_interval = 3 + max_sleep_interval = 20 max_wait_time = 300 diff_url = f"{self.host}/datadiffs/{diff_id}/overview" @@ -260,13 +260,15 @@ def poll_data_diff_results(self, diff_id: int) -> TCloudApiDataDiffSummaryResult if response_json["status"] == "success": summary_results = response_json elif response_json["status"] == "failed": - raise Exception(f"Diff failed: {str(response_json)}") + raise DataDiffCloudDiffFailed(f"Diff failed: {str(response_json)}") if time.monotonic() - start_time > max_wait_time: - raise Exception(f"Timed out waiting for diff results. Please, go to the UI for details: {diff_url}") + raise DataDiffCloudDiffTimedOut( + f"Timed out waiting for diff results. Please, go to the UI for details: {diff_url}" + ) time.sleep(sleep_interval) - sleep_interval = min(sleep_interval * 2, max_sleep_interval) + sleep_interval = min(sleep_interval + 1, max_sleep_interval) return TCloudApiDataDiffSummaryResult.from_orm(summary_results) diff --git a/data_diff/errors.py b/data_diff/errors.py index 9af915bf..dd1c887e 100644 --- a/data_diff/errors.py +++ b/data_diff/errors.py @@ -60,3 +60,11 @@ class DataDiffNoDatasourceIdError(Exception): class DataDiffDatasourceIdNotFoundError(Exception): "Raised when using --cloud but the datasource_id is not found for a particular org." + + +class DataDiffCloudDiffFailed(Exception): + "Raised when using --cloud and the remote diff fails." + + +class DataDiffCloudDiffTimedOut(Exception): + "Raised when using --cloud and the diff did not return finish before the timeout value."