Merge branch 'master' into mask-creds-local-mode

Author: ajaykarpur

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## v2.25.2 (2021-02-25)
+
+### Bug Fixes and Other Changes
+
+ * Use the output path to store the Clarify config file
+ * feature group should ignore nan values
+ * ignore failing smdataparallel test
+ * Add tests for Training job & Transform job in visualizer
+ * visualizer for pipeline processing job steps
+
+### Documentation Changes
+
+ * update doc for Elastic Inference MXNet 1.7.0
+
 ## v2.25.1 (2021-02-20)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.25.2.dev0
+2.25.3.dev0

src/sagemaker/clarify.py

@@ -21,7 +21,7 @@
 import tempfile
 
 from sagemaker.processing import ProcessingInput, ProcessingOutput, Processor
-from sagemaker import image_uris, utils
+from sagemaker import image_uris, s3, utils
 
 
 class DataConfig:
@@ -405,9 +405,15 @@ def _run(
             analysis_config_file = os.path.join(tmpdirname, "analysis_config.json")
             with open(analysis_config_file, "w") as f:
                 json.dump(analysis_config, f)
+            s3_analysis_config_file = _upload_analysis_config(
+                analysis_config_file,
+                data_config.s3_output_path,
+                self.sagemaker_session,
+                kms_key,
+            )
             config_input = ProcessingInput(
                 input_name="analysis_config",
-                source=analysis_config_file,
+                source=s3_analysis_config_file,
                 destination=self._CLARIFY_CONFIG_INPUT,
                 s3_data_type="S3Prefix",
                 s3_input_mode="File",
@@ -638,6 +644,30 @@ def run_explainability(
         self._run(data_config, analysis_config, wait, logs, job_name, kms_key)
 
 
+def _upload_analysis_config(analysis_config_file, s3_output_path, sagemaker_session, kms_key):
+    """Uploads the local analysis_config_file to the s3_output_path.
+
+    Args:
+        analysis_config_file (str): File path to the local analysis config file.
+        s3_output_path (str): S3 prefix to store the analysis config file.
+        sagemaker_session (:class:`~sagemaker.session.Session`):
+            Session object which manages interactions with Amazon SageMaker and
+            any other AWS services needed. If not specified, the processor creates
+            one using the default AWS configuration chain.
+        kms_key (str): The ARN of the KMS key that is used to encrypt the
+            user code file (default: None).
+
+    Returns:
+        The S3 uri of the uploaded file.
+    """
+    return s3.S3Uploader.upload(
+        local_path=analysis_config_file,
+        desired_s3_uri=s3_output_path,
+        sagemaker_session=sagemaker_session,
+        kms_key=kms_key,
+    )
+
+
 def _set(value, key, dictionary):
     """Sets dictionary[key] = value if value is not None."""
     if value is not None:

src/sagemaker/feature_store/feature_group.py

@@ -185,6 +185,7 @@ def _ingest_single_batch(
                     feature_name=data_frame.columns[index], value_as_string=str(row[index])
                 )
                 for index in range(len(row))
+                if pd.notna(row[index])
             ]
             sagemaker_session.put_record(
                 feature_group_name=feature_group_name, record=[value.to_dict() for value in record]

src/sagemaker/image_uri_config/mxnet.json

@@ -694,7 +694,8 @@
         "version_aliases": {
             "1.3": "1.3.0",
             "1.4": "1.4.1",
-            "1.5": "1.5.1"
+            "1.5": "1.5.1",
+            "1.7": "1.7.0"
         },
         "versions": {
             "1.3.0": {
@@ -816,6 +817,36 @@
                 },
                 "repository": "mxnet-inference-eia",
                 "py_versions": ["py2", "py3"]
+            },
+            "1.7.0": {
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "mxnet-inference-eia",
+                "py_versions": ["py3"]
             }
         }
     }

src/sagemaker/image_uri_config/neo-pytorch.json

@@ -2,14 +2,73 @@
     "processors": ["cpu", "gpu"],
     "scope": ["inference"],
     "version_aliases": {
-        "0.4.0": "1.4.0",
-        "1.0.0": "1.4.0",
-        "1.1.0": "1.4.0",
-        "1.2.0": "1.4.0",
-        "1.3.0": "1.4.0"
+        "0.4.0": "1.4",
+        "1.0.0": "1.4",
+        "1.1.0": "1.4",
+        "1.2.0": "1.4",
+        "1.3.0": "1.4",
+        "1.4.0": "1.4"
     },
     "versions": {
-        "1.4.0": {
+        "1.4": {
+            "py_versions": ["py3"],
+            "registries": {
+                "af-south-1": "774647643957",
+                "ap-east-1": "110948597952",
+                "ap-northeast-1": "941853720454",
+                "ap-northeast-2": "151534178276",
+                "ap-south-1": "763008648453",
+                "ap-southeast-1": "324986816169",
+                "ap-southeast-2": "355873309152",
+                "ca-central-1": "464438896020",
+                "cn-north-1": "472730292857",
+                "cn-northwest-1": "474822919863",
+                "eu-central-1": "746233611703",
+                "eu-north-1": "601324751636",
+                "eu-south-1": "966458181534",
+                "eu-west-1": "802834080501",
+                "eu-west-2": "205493899709",
+                "eu-west-3": "254080097072",
+                "me-south-1": "836785723513",
+                "sa-east-1": "756306329178",
+                "us-east-1": "785573368785",
+                "us-east-2": "007439368137",
+                "us-gov-west-1": "263933020539",
+                "us-west-1": "710691900526",
+                "us-west-2": "301217895009"
+            },
+            "repository": "sagemaker-inference-pytorch"
+        },
+        "1.5": {
+            "py_versions": ["py3"],
+            "registries": {
+                "af-south-1": "774647643957",
+                "ap-east-1": "110948597952",
+                "ap-northeast-1": "941853720454",
+                "ap-northeast-2": "151534178276",
+                "ap-south-1": "763008648453",
+                "ap-southeast-1": "324986816169",
+                "ap-southeast-2": "355873309152",
+                "ca-central-1": "464438896020",
+                "cn-north-1": "472730292857",
+                "cn-northwest-1": "474822919863",
+                "eu-central-1": "746233611703",
+                "eu-north-1": "601324751636",
+                "eu-south-1": "966458181534",
+                "eu-west-1": "802834080501",
+                "eu-west-2": "205493899709",
+                "eu-west-3": "254080097072",
+                "me-south-1": "836785723513",
+                "sa-east-1": "756306329178",
+                "us-east-1": "785573368785",
+                "us-east-2": "007439368137",
+                "us-gov-west-1": "263933020539",
+                "us-west-1": "710691900526",
+                "us-west-2": "301217895009"
+            },
+            "repository": "sagemaker-inference-pytorch"
+        },
+        "1.6": {
             "py_versions": ["py3"],
             "registries": {
                 "af-south-1": "774647643957",

src/sagemaker/lineage/action.py

@@ -186,7 +186,7 @@ def create(
         return super(Action, cls)._construct(
             cls._boto_create_method,
             action_name=action_name,
-            source=_api_types.ContextSource(source_uri=source_uri, source_type=source_type),
+            source=_api_types.ActionSource(source_uri=source_uri, source_type=source_type),
             action_type=action_type,
             description=description,
             status=status,

src/sagemaker/lineage/artifact.py

@@ -258,7 +258,7 @@ def create(
         return super(Artifact, cls)._construct(
             cls._boto_create_method,
             artifact_name=artifact_name,
-            source=_api_types.ContextSource(source_uri=source_uri, source_types=source_types),
+            source=_api_types.ArtifactSource(source_uri=source_uri, source_types=source_types),
             artifact_type=artifact_type,
             properties=properties,
             tags=tags,

src/sagemaker/model.py

@@ -16,6 +16,7 @@
 import json
 import logging
 import os
+import re
 
 import sagemaker
 from sagemaker import (
@@ -398,6 +399,7 @@ def _compilation_job_config(
         target_platform_arch=None,
         target_platform_accelerator=None,
         compiler_options=None,
+        framework_version=None,
     ):
         """Placeholder Docstring"""
         input_model_config = {
@@ -407,6 +409,14 @@ def _compilation_job_config(
             else input_shape,
             "Framework": framework.upper(),
         }
+
+        if (
+            framework.lower() == "pytorch"
+            and re.match("(?=^ml_)(?!ml_inf)", target_instance_type) is not None
+            and framework_version is not None
+        ):
+            input_model_config["FrameworkVersion"] = utils.get_short_version(framework_version)
+
         role = self.sagemaker_session.expand_role(role)
         output_model_config = {
             "S3OutputLocation": output_path,
@@ -572,7 +582,8 @@ def compile(
             framework (str): The framework that is used to train the original
                 model. Allowed values: 'mxnet', 'tensorflow', 'keras', 'pytorch',
                 'onnx', 'xgboost'
-            framework_version (str):
+            framework_version (str): The version of framework, for example:
+                '1.5' for PyTorch
             target_platform_os (str): Target Platform OS, for example: 'LINUX'.
                 For allowed strings see
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
@@ -626,11 +637,11 @@ def compile(
             target_platform_arch,
             target_platform_accelerator,
             compiler_options,
+            framework_version,
         )
         self.sagemaker_session.compile_model(**config)
         job_status = self.sagemaker_session.wait_for_compilation_job(job_name)
         self.model_data = job_status["ModelArtifacts"]["S3ModelArtifacts"]
-
         if target_instance_family is not None:
             if target_instance_family.startswith("ml_"):
                 self.image_uri = self._compilation_image_uri(

src/sagemaker/model_monitor/model_monitoring.py

@@ -238,7 +238,7 @@ def create_monitoring_schedule(
                 schedule.
             statistics (sagemaker.model_monitor.Statistic or str): If provided alongside
                 constraints, these will be used for monitoring the endpoint. This can be a
-                sagemaker.model_monitor.Constraints object or an S3 uri pointing to a constraints
+                sagemaker.model_monitor.Statistic object or an S3 uri pointing to a statistic
                 JSON file.
             constraints (sagemaker.model_monitor.Constraints or str): If provided alongside
                 statistics, these will be used for monitoring the endpoint. This can be a
@@ -352,7 +352,7 @@ def update_monitoring_schedule(
                 schedule.
             statistics (sagemaker.model_monitor.Statistic or str): If provided alongside
                 constraints, these will be used for monitoring the endpoint. This can be a
-                sagemaker.model_monitor.Constraints object or an S3 uri pointing to a constraints
+                sagemaker.model_monitor.Statistics object or an S3 uri pointing to a statistics
                 JSON file.
             constraints (sagemaker.model_monitor.Constraints or str): If provided alongside
                 statistics, these will be used for monitoring the endpoint. This can be a
@@ -1552,7 +1552,7 @@ def create_monitoring_schedule(
                 JSON file.
             statistics (sagemaker.model_monitor.Statistic or str): If provided alongside
                 constraints, these will be used for monitoring the endpoint. This can be a
-                sagemaker.model_monitor.Constraints object or an s3_uri pointing to a constraints
+                sagemaker.model_monitor.Statistics object or an s3_uri pointing to a statistics
                 JSON file.
             monitor_schedule_name (str): Schedule name. If not specified, the processor generates
                 a default job name, based on the image name and current timestamp.
@@ -1657,7 +1657,7 @@ def update_monitoring_schedule(
                 statistics json files.
             statistics (sagemaker.model_monitor.Statistic or str): If provided alongside
                 constraints, these will be used for monitoring the endpoint. This can be a
-                sagemaker.model_monitor.Constraints object or an S3 uri pointing to a constraints
+                sagemaker.model_monitor.Statistics object or an S3 uri pointing to a statistics
                 JSON file.
             constraints (sagemaker.model_monitor.Constraints or str): If provided alongside
                 statistics, these will be used for monitoring the endpoint. This can be a
@@ -1843,7 +1843,7 @@ def _update_data_quality_monitoring_schedule(
                 to a constraints JSON file.
             statistics (sagemaker.model_monitor.Statistic or str): If provided alongside
                 constraints, these will be used for monitoring the endpoint. This can be a
-                sagemaker.model_monitor.Constraints object or an S3 uri pointing to a constraints
+                sagemaker.model_monitor.Statistics object or an S3 uri pointing to a statistics
                 JSON file.
             schedule_cron_expression (str): The cron expression that dictates the frequency that
                 this job run. See sagemaker.model_monitor.CronExpressionGenerator for valid

tests/conftest.py

@@ -157,8 +157,16 @@ def mxnet_training_py_version(mxnet_training_version, request):
 
 
 @pytest.fixture(scope="module", params=["py2", "py3"])
-def mxnet_eia_py_version(request):
-    return request.param
+def mxnet_eia_py_version(mxnet_eia_version, request):
+    if Version(mxnet_eia_version) < Version("1.7.0"):
+        return request.param
+    else:
+        return "py3"
+
+
+@pytest.fixture(scope="module")
+def mxnet_eia_latest_py_version():
+    return "py3"
 
 
 @pytest.fixture(scope="module", params=["py2", "py3"])
@@ -182,6 +190,26 @@ def pytorch_eia_py_version():
     return "py3"
 
 
+@pytest.fixture(scope="module")
+def neo_pytorch_latest_py_version():
+    return "py3"
+
+
+@pytest.fixture(scope="module")
+def neo_pytorch_compilation_job_name():
+    return utils.name_from_base("pytorch-neo-model")
+
+
+@pytest.fixture(scope="module")
+def neo_pytorch_target_device():
+    return "ml_c5"
+
+
+@pytest.fixture(scope="module")
+def neo_pytorch_cpu_instance_type():
+    return "ml.c5.xlarge"
+
+
 @pytest.fixture(scope="module")
 def xgboost_framework_version(xgboost_version):
     if xgboost_version in ("1", "latest"):