Merge branch 'master' into mask-creds-local-mode

Author: jmahlik

CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## v2.25.1 (2021-02-20)
+
+### Bug Fixes and Other Changes
+
+ * Add tests for visualizer to improve test coverage
+
+### Documentation Changes
+
+ * specify correct return type
+
+### Testing and Release Infrastructure
+
+ * rename canary_quick pytest mark to release
+
+## v2.25.0 (2021-02-19)
+
+### Features
+
+ * Enable step caching
+ * Add other Neo supported regions for Inferentia inference images
+
+### Bug Fixes and Other Changes
+
+ * remove FailStep from pipelines
+ * use sagemaker_session in workflow tests
+ * use ECR public for multidatamodel tests
+ * add the mapping from py3 to cuda11 images
+ * Add 30s cap time for tag tests
+ * add build spec for slow tests
+ * mark top 10 slow tests
+ * remove slow test_run_xxx_monitor_baseline tests
+ * pin astroid to 2.4.2
+
+### Testing and Release Infrastructure
+
+ * unmark more flaky integ tests
+ * remove canary_quick pytest mark from flaky/unnecessary tests
+ * remove python3.8 from buildspec
+ * remove py38 tox env
+ * fix release buildspec typo
+ * unblock regional release builds
+ * lower test TPS for experiment analytics
+ * move package preparation and publishing to the deploy step
+
 ## v2.24.5 (2021-02-12)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.24.6.dev0
+2.25.2.dev0

buildspec-localmodetests.yml

@@ -11,5 +11,5 @@ phases:
 
       # local mode tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
-      - ./ci-scripts/displaytime.sh 'py38 local mode' $start_time
+      - execute-command-if-has-matching-changes "tox -e py37 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
+      - ./ci-scripts/displaytime.sh 'py37 local mode' $start_time

buildspec-release.yml

@@ -15,7 +15,7 @@ phases:
       # run unit tests
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py37,py38 -- tests/unit
+        tox -e py36,py37 -- tests/unit
 
       # run a subset of the integration tests
-      - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2
+      - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m "not (local_mode or slow_test)" -n 32 --boxed --reruns 2

buildspec-slowtests.yml

@@ -1,11 +1,15 @@
 version: 0.2
 
 phases:
+  pre_build:
+    commands:
+      - start-dockerd
+
   build:
     commands:
       - IGNORE_COVERAGE=-
 
       # slow tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m slow_test -n 10 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
-      - ./ci-scripts/displaytime.sh 'py38 slow tests' $start_time
+      - execute-command-if-has-matching-changes "tox -e py37 -- tests/integ -m slow_test -n 16 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
+      - ./ci-scripts/displaytime.sh 'py37 slow tests' $start_time

buildspec-unittests.yml

@@ -18,5 +18,5 @@ phases:
       - start_time=`date +%s`
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py37,py38 --parallel all -- tests/unit
-      - ./ci-scripts/displaytime.sh 'py36,py37,py38 unit' $start_time
+        tox -e py36,py37 --parallel all -- tests/unit
+      - ./ci-scripts/displaytime.sh 'py36,py37 unit' $start_time

buildspec.yml

@@ -11,13 +11,13 @@ phases:
 
       # run integration tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "python3.8 -u ci-scripts/queue_build.py" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"
+      - execute-command-if-has-matching-changes "python3.7 -u ci-scripts/queue_build.py" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"
       - ./ci-scripts/displaytime.sh 'build queue' $start_time
 
       - start_time=`date +%s`
       - |
-        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron and not slow_test\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
-      - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time
+        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py37 -- tests/integ -m \"not local_mode and not cron and not slow_test\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
+      - ./ci-scripts/displaytime.sh 'py37 tests/integ' $start_time
 
   post_build:
     finally:

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -110,8 +110,6 @@ Steps
 
 .. autoclass:: sagemaker.workflow.steps.ProcessingStep
 
-.. autoclass:: sagemaker.workflow.steps.FailStep
-
 Utilities
 ---------
 

setup.py

@@ -66,6 +66,7 @@ def read_version():
         "pytest<6.1.0",
         "pytest-cov",
         "pytest-rerunfailures",
+        "pytest-timeout",
         "pytest-xdist",
         "mock",
         "contextlib2",

src/sagemaker/image_uris.py

@@ -98,6 +98,8 @@ def retrieve(
             "mxnet-1.8.0-gpu-py37": "cu110-ubuntu16.04",
             "pytorch-1.6-gpu-py36": "cu110-ubuntu18.04-v3",
             "pytorch-1.6.0-gpu-py36": "cu110-ubuntu18.04",
+            "pytorch-1.6-gpu-py3": "cu110-ubuntu18.04-v3",
+            "pytorch-1.6.0-gpu-py3": "cu110-ubuntu18.04",
         }
         key = "-".join([framework, tag])
         if key in container_versions:

src/sagemaker/model.py

@@ -146,7 +146,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         if self.model_data is None:
             raise ValueError("SageMaker Model Package cannot be created without model data.")

src/sagemaker/mxnet/model.py

@@ -183,7 +183,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         instance_type = inference_instances[0]
         self._init_sagemaker_session_if_does_not_exist(instance_type)

src/sagemaker/pytorch/model.py

@@ -182,7 +182,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         instance_type = inference_instances[0]
         self._init_sagemaker_session_if_does_not_exist(instance_type)

src/sagemaker/tensorflow/model.py

@@ -226,7 +226,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         instance_type = inference_instances[0]
         self._init_sagemaker_session_if_does_not_exist(instance_type)

src/sagemaker/workflow/steps.py

@@ -50,7 +50,6 @@ class StepTypeEnum(Enum, metaclass=DefaultEnumMeta):
 
     CONDITION = "Condition"
     CREATE_MODEL = "Model"
-    FAIL = "Fail"
     PROCESSING = "Processing"
     REGISTER_MODEL = "RegisterModel"
     TRAINING = "Training"
@@ -93,6 +92,38 @@ def ref(self) -> Dict[str, str]:
         return {"Name": self.name}
 
 
+@attr.s
+class CacheConfig:
+    """Configuration class to enable caching in pipeline workflow.
+
+    If caching is enabled, the pipeline attempts to find a previous execution of a step
+    that was called with the same arguments. Step caching only considers successful execution.
+    If a successful previous execution is found, the pipeline propagates the values
+    from previous execution rather than recomputing the step. When multiple successful executions
+    exist within the timeout period, it uses the result for the most recent successful execution.
+
+
+    Attributes:
+        enable_caching (bool): To enable step caching. Defaults to `False`.
+        expire_after (str): If step caching is enabled, a timeout also needs to defined.
+            It defines how old a previous execution can be to be considered for reuse.
+            Value should be an ISO 8601 duration string. Defaults to `None`.
+    """
+
+    enable_caching: bool = attr.ib(default=False)
+    expire_after = attr.ib(
+        default=None, validator=attr.validators.optional(attr.validators.instance_of(str))
+    )
+
+    @property
+    def config(self):
+        """Configures caching in pipeline steps."""
+        config = {"Enabled": self.enable_caching}
+        if self.expire_after is not None:
+            config["ExpireAfter"] = self.expire_after
+        return {"CacheConfig": config}
+
+
 class TrainingStep(Step):
     """Training step for workflow."""
 
@@ -101,6 +132,7 @@ def __init__(
         name: str,
         estimator: EstimatorBase,
         inputs: TrainingInput = None,
+        cache_config: CacheConfig = None,
     ):
         """Construct a TrainingStep, given an `EstimatorBase` instance.
 
@@ -111,14 +143,15 @@ def __init__(
             name (str): The name of the training step.
             estimator (EstimatorBase): A `sagemaker.estimator.EstimatorBase` instance.
             inputs (TrainingInput): A `sagemaker.inputs.TrainingInput` instance. Defaults to `None`.
+            cache_config (CacheConfig):  A `sagemaker.workflow.steps.CacheConfig` instance.
         """
         super(TrainingStep, self).__init__(name, StepTypeEnum.TRAINING)
         self.estimator = estimator
         self.inputs = inputs
-
         self._properties = Properties(
             path=f"Steps.{name}", shape_name="DescribeTrainingJobResponse"
         )
+        self.cache_config = cache_config
 
     @property
     def arguments(self) -> RequestType:
@@ -145,6 +178,14 @@ def properties(self):
         """A Properties object representing the DescribeTrainingJobResponse data model."""
         return self._properties
 
+    def to_request(self) -> RequestType:
+        """Updates the dictionary with cache configuration."""
+        request_dict = super().to_request()
+        if self.cache_config:
+            request_dict.update(self.cache_config.config)
+
+        return request_dict
+
 
 class CreateModelStep(Step):
     """CreateModel step for workflow."""
@@ -208,6 +249,7 @@ def __init__(
         name: str,
         transformer: Transformer,
         inputs: TransformInput,
+        cache_config: CacheConfig = None,
     ):
         """Constructs a TransformStep, given an `Transformer` instance.
 
@@ -218,11 +260,12 @@ def __init__(
             name (str): The name of the transform step.
             transformer (Transformer): A `sagemaker.transformer.Transformer` instance.
             inputs (TransformInput): A `sagemaker.inputs.TransformInput` instance.
+            cache_config (CacheConfig):  A `sagemaker.workflow.steps.CacheConfig` instance.
         """
         super(TransformStep, self).__init__(name, StepTypeEnum.TRANSFORM)
         self.transformer = transformer
         self.inputs = inputs
-
+        self.cache_config = cache_config
         self._properties = Properties(
             path=f"Steps.{name}", shape_name="DescribeTransformJobResponse"
         )
@@ -258,6 +301,14 @@ def properties(self):
         """A Properties object representing the DescribeTransformJobResponse data model."""
         return self._properties
 
+    def to_request(self) -> RequestType:
+        """Updates the dictionary with cache configuration."""
+        request_dict = super().to_request()
+        if self.cache_config:
+            request_dict.update(self.cache_config.config)
+
+        return request_dict
+
 
 class ProcessingStep(Step):
     """Processing step for workflow."""
@@ -271,6 +322,7 @@ def __init__(
         job_arguments: List[str] = None,
         code: str = None,
         property_files: List[PropertyFile] = None,
+        cache_config: CacheConfig = None,
     ):
         """Construct a ProcessingStep, given a `Processor` instance.
 
@@ -290,6 +342,7 @@ def __init__(
                 script to run. Defaults to `None`.
             property_files (List[PropertyFile]): A list of property files that workflow looks
                 for and resolves from the configured processing output list.
+            cache_config (CacheConfig):  A `sagemaker.workflow.steps.CacheConfig` instance.
         """
         super(ProcessingStep, self).__init__(name, StepTypeEnum.PROCESSING)
         self.processor = processor
@@ -306,6 +359,7 @@ def __init__(
         self._properties = Properties(
             path=f"Steps.{name}", shape_name="DescribeProcessingJobResponse"
         )
+        self.cache_config = cache_config
 
     @property
     def arguments(self) -> RequestType:
@@ -336,49 +390,10 @@ def properties(self):
     def to_request(self) -> RequestType:
         """Get the request structure for workflow service calls."""
         request_dict = super(ProcessingStep, self).to_request()
+        if self.cache_config:
+            request_dict.update(self.cache_config.config)
         if self.property_files:
             request_dict["PropertyFiles"] = [
                 property_file.expr for property_file in self.property_files
             ]
         return request_dict
-
-
-class FailStep(Step):
-    """Pipeline step to indicate failure."""
-
-    def __init__(self, name: str = "Fail"):
-        """Construct a FailStep.
-
-        Causes the pipeline execution to terminate in a failed state.
-
-        Args:
-            name (str): The name of the step.
-        """
-        super(FailStep, self).__init__(name, StepTypeEnum.FAIL)
-        root_path = f"Steps.{name}"
-        root_prop = Properties(path=root_path)
-        root_prop.__dict__["Fail"] = Properties(f"{root_path}.Fail")
-        self._properties = root_prop
-
-    @property
-    def arguments(self) -> RequestType:
-        """The arguments to the particular step service call."""
-        return {}
-
-    @property
-    def properties(self):
-        """The properties of the particular step."""
-        return self._properties
-
-    def to_request(self) -> RequestType:
-        """Get the request structure for workflow service calls."""
-        return {
-            "Name": self.name,
-            "Type": self.step_type.value,
-            "Arguments": self.arguments,
-        }
-
-    @property
-    def ref(self) -> Dict[str, str]:
-        """Get a reference dict for steps"""
-        return {"Name": self.name}

tests/data/multimodel/container/Dockerfile

@@ -1,4 +1,4 @@
-FROM 142577830533.dkr.ecr.us-east-2.amazonaws.com/ubuntu:16.04
+FROM public.ecr.aws/ubuntu/ubuntu:18.04
 
 # Set a docker label to advertise multi-model support on the container
 LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
@@ -15,7 +15,7 @@ RUN apt-get update && \
     curl \
     vim \
     && rm -rf /var/lib/apt/lists/* \
-    && curl -O https://bootstrap.pypa.io/3.5/get-pip.py \
+    && curl -O https://bootstrap.pypa.io/get-pip.py \
     && python3 get-pip.py
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1