fix: when customer role does not have permission to read logs from CW, default to standard logging - Galactus (#1387)

Co-authored-by: Raymond Liu <[email protected]>

Author: 2 people

src/sagemaker/session.py

@@ -4934,7 +4934,8 @@ def wait_for_endpoint(self, endpoint, poll=DEFAULT_EP_POLL, live_logging=False):
         Returns:
             dict: Return value from the ``DescribeEndpoint`` API.
         """
-        if not live_logging:
+
+        if not live_logging or not _has_permission_for_live_logging(self.boto_session, endpoint):
             desc = _wait_until(lambda: _deploy_done(self.sagemaker_client, endpoint), poll)
         else:
             cloudwatch_client = self.boto_session.client("logs")
@@ -7614,5 +7615,25 @@ def _flush_log_streams(
         sys.stdout.flush()
 
 
+def _has_permission_for_live_logging(boto_session, endpoint_name) -> bool:
+    """Validate if customer's role has the right permission to access logs from CloudWatch"""
+    try:
+        cloudwatch_client = boto_session.client("logs")
+        cloudwatch_client.filter_log_events(
+            logGroupName=f"/aws/sagemaker/Endpoints/{endpoint_name}",
+            logStreamNamePrefix="AllTraffic/",
+        )
+        return True
+    except ClientError as e:
+        if e.response["Error"]["Code"] == "AccessDeniedException":
+            LOGGER.warning(
+                ("Failed to enable live logging: %s. Fallback to default logging..."),
+                e,
+            )
+
+            return False
+        return True
+
+
 s3_input = deprecated_class(TrainingInput, "sagemaker.session.s3_input")
 ShuffleConfig = deprecated_class(ShuffleConfig, "sagemaker.session.ShuffleConfig")