From 222a56e59e9b182f090e407c9f2a8d0e443dacaa Mon Sep 17 00:00:00 2001 From: Nadia Yakimakha <32335935+nadiaya@users.noreply.github.com> Date: Thu, 4 Jun 2020 17:05:37 -0700 Subject: [PATCH 01/18] feature: Use boto3 DEFAULT_SESSION when no boto3 session specified. (#1545) --- src/sagemaker/session.py | 2 +- tests/unit/test_session.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sagemaker/session.py b/src/sagemaker/session.py index c58801a886..6fac50ad29 100644 --- a/src/sagemaker/session.py +++ b/src/sagemaker/session.py @@ -123,7 +123,7 @@ def _initialize(self, boto_session, sagemaker_client, sagemaker_runtime_client): Creates or uses a boto_session, sagemaker_client and sagemaker_runtime_client. Sets the region_name. """ - self.boto_session = boto_session or boto3.Session() + self.boto_session = boto_session or boto3.DEFAULT_SESSION or boto3.Session() self._region_name = self.boto_session.region_name if self._region_name is None: diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 67fffc118c..5aaa3f0af0 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -52,6 +52,18 @@ def boto_session(): return boto_mock +@patch("boto3.DEFAULT_SESSION") +def test_default_session(boto3_default_session): + sess = Session() + assert sess.boto_session is boto3_default_session + + +@patch("boto3.Session") +def test_new_session_created(boto3_session): + sess = Session() + assert sess.boto_session is boto3_session.return_value + + def test_process(boto_session): session = Session(boto_session) From b3f51a84d86fc373ed2f5cdc257ddc69df92d3ef Mon Sep 17 00:00:00 2001 From: Lauren Yu <6631887+laurenyu@users.noreply.github.com> Date: Mon, 8 Jun 2020 16:27:23 -0700 Subject: [PATCH 02/18] change: remove v2 Session warnings (#1556) These changes have been deprioritized. --- src/sagemaker/session.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/sagemaker/session.py b/src/sagemaker/session.py index 6fac50ad29..7bee7c332f 100644 --- a/src/sagemaker/session.py +++ b/src/sagemaker/session.py @@ -181,12 +181,6 @@ def upload_data(self, path, bucket=None, key_prefix="data", extra_args=None): ``s3://{bucket name}/{key_prefix}``. """ # Generate a tuple for each file that we want to upload of the form (local_path, s3_key). - LOGGER.warning( - "'upload_data' method will be deprecated in favor of 'S3Uploader' class " - "(https://sagemaker.readthedocs.io/en/stable/s3.html#sagemaker.s3.S3Uploader) " - "in SageMaker Python SDK v2." - ) - files = [] key_suffix = None if os.path.isdir(path): @@ -236,12 +230,6 @@ def upload_string_as_file_body(self, body, bucket, key, kms_key=None): str: The S3 URI of the uploaded file. The URI format is: ``s3://{bucket name}/{key}``. """ - LOGGER.warning( - "'upload_string_as_file_body' method will be deprecated in favor of 'S3Uploader' class " - "(https://sagemaker.readthedocs.io/en/stable/s3.html#sagemaker.s3.S3Uploader) " - "in SageMaker Python SDK v2." - ) - if self.s3_resource is None: s3 = self.boto_session.resource("s3", region_name=self.boto_region_name) else: @@ -3335,7 +3323,6 @@ def get_execution_role(sagemaker_session=None): Returns: (str): The role ARN """ - if not sagemaker_session: sagemaker_session = Session() arn = sagemaker_session.get_caller_identity_arn() From 13c1e5fa9e0f593d0250788f12861e1b55a22cfd Mon Sep 17 00:00:00 2001 From: ci Date: Tue, 9 Jun 2020 20:04:35 +0000 Subject: [PATCH 03/18] prepare release v1.61.0 --- CHANGELOG.md | 12 ++++++++++++ VERSION | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c097e85827..56f9bb21b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## v1.61.0 (2020-06-09) + +### Features + + * Use boto3 DEFAULT_SESSION when no boto3 session specified. + +### Bug Fixes and Other Changes + + * remove v2 Session warnings + * upgrade smdebug-rulesconfig to 0.1.4 + * explicitly handle arguments in create_model for sklearn and xgboost + ## v1.60.2 (2020-05-29) ### Bug Fixes and Other Changes diff --git a/VERSION b/VERSION index de928bad6d..91951fd8ad 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.60.3.dev0 +1.61.0 From a84152a9bd564d0ae7f27d1afb39460881328661 Mon Sep 17 00:00:00 2001 From: ci Date: Tue, 9 Jun 2020 20:35:27 +0000 Subject: [PATCH 04/18] update development version to v1.61.1.dev0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 91951fd8ad..16cd31157e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.61.0 +1.61.1.dev0 From 5d57f7bcc88c317acfd484bf52396c20b1e18ee0 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 10 Jun 2020 10:36:18 -0700 Subject: [PATCH 05/18] docs: workflows navigation (#1563) * documentation: workflow nav addition * documentation: add index page for workflows folder --- doc/index.rst | 9 ++++----- doc/{ => workflows}/airflow/index.rst | 0 .../airflow/sagemaker.workflow.airflow.rst | 0 doc/{ => workflows}/airflow/using_workflow.rst | 0 doc/workflows/index.rst | 11 +++++++++++ ..._sagemaker_components_for_kubeflow_pipelines.rst | 0 .../kubernetes/amazon_sagemaker_jobs.rst | 0 .../amazon_sagemaker_operators_for_kubernetes.rst | 0 ...aker_operators_for_kubernetes_authentication.png | Bin doc/{ => workflows}/kubernetes/index.rst | 0 .../using_amazon_sagemaker_components.rst | 0 11 files changed, 15 insertions(+), 5 deletions(-) rename doc/{ => workflows}/airflow/index.rst (100%) rename doc/{ => workflows}/airflow/sagemaker.workflow.airflow.rst (100%) rename doc/{ => workflows}/airflow/using_workflow.rst (100%) create mode 100644 doc/workflows/index.rst rename doc/{ => workflows}/kubernetes/amazon_sagemaker_components_for_kubeflow_pipelines.rst (100%) rename doc/{ => workflows}/kubernetes/amazon_sagemaker_jobs.rst (100%) rename doc/{ => workflows}/kubernetes/amazon_sagemaker_operators_for_kubernetes.rst (100%) rename doc/{ => workflows}/kubernetes/amazon_sagemaker_operators_for_kubernetes_authentication.png (100%) rename doc/{ => workflows}/kubernetes/index.rst (100%) rename doc/{ => workflows}/kubernetes/using_amazon_sagemaker_components.rst (100%) diff --git a/doc/index.rst b/doc/index.rst index 5a698ae80a..db8e8e166b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -53,15 +53,14 @@ Amazon SageMaker provides implementations of some common machine learning algori ************* -Orchestration +Workflows ************* -Orchestrate your SageMaker training and inference jobs with Kubernetes and Airflow. +Orchestrate your SageMaker training and inference workflows with Airflow and Kubernetes. .. toctree:: - :maxdepth: 1 + :maxdepth: 2 - kubernetes/index - airflow/index + workflows/index ************************* diff --git a/doc/airflow/index.rst b/doc/workflows/airflow/index.rst similarity index 100% rename from doc/airflow/index.rst rename to doc/workflows/airflow/index.rst diff --git a/doc/airflow/sagemaker.workflow.airflow.rst b/doc/workflows/airflow/sagemaker.workflow.airflow.rst similarity index 100% rename from doc/airflow/sagemaker.workflow.airflow.rst rename to doc/workflows/airflow/sagemaker.workflow.airflow.rst diff --git a/doc/airflow/using_workflow.rst b/doc/workflows/airflow/using_workflow.rst similarity index 100% rename from doc/airflow/using_workflow.rst rename to doc/workflows/airflow/using_workflow.rst diff --git a/doc/workflows/index.rst b/doc/workflows/index.rst new file mode 100644 index 0000000000..cbd4cfbc86 --- /dev/null +++ b/doc/workflows/index.rst @@ -0,0 +1,11 @@ +########## +Workflows +########## + +The SageMaker Python SDK supports managed training and inference for a variety of machine learning frameworks: + +.. toctree:: + :maxdepth: 2 + + airflow/index + kubernetes/index diff --git a/doc/kubernetes/amazon_sagemaker_components_for_kubeflow_pipelines.rst b/doc/workflows/kubernetes/amazon_sagemaker_components_for_kubeflow_pipelines.rst similarity index 100% rename from doc/kubernetes/amazon_sagemaker_components_for_kubeflow_pipelines.rst rename to doc/workflows/kubernetes/amazon_sagemaker_components_for_kubeflow_pipelines.rst diff --git a/doc/kubernetes/amazon_sagemaker_jobs.rst b/doc/workflows/kubernetes/amazon_sagemaker_jobs.rst similarity index 100% rename from doc/kubernetes/amazon_sagemaker_jobs.rst rename to doc/workflows/kubernetes/amazon_sagemaker_jobs.rst diff --git a/doc/kubernetes/amazon_sagemaker_operators_for_kubernetes.rst b/doc/workflows/kubernetes/amazon_sagemaker_operators_for_kubernetes.rst similarity index 100% rename from doc/kubernetes/amazon_sagemaker_operators_for_kubernetes.rst rename to doc/workflows/kubernetes/amazon_sagemaker_operators_for_kubernetes.rst diff --git a/doc/kubernetes/amazon_sagemaker_operators_for_kubernetes_authentication.png b/doc/workflows/kubernetes/amazon_sagemaker_operators_for_kubernetes_authentication.png similarity index 100% rename from doc/kubernetes/amazon_sagemaker_operators_for_kubernetes_authentication.png rename to doc/workflows/kubernetes/amazon_sagemaker_operators_for_kubernetes_authentication.png diff --git a/doc/kubernetes/index.rst b/doc/workflows/kubernetes/index.rst similarity index 100% rename from doc/kubernetes/index.rst rename to doc/workflows/kubernetes/index.rst diff --git a/doc/kubernetes/using_amazon_sagemaker_components.rst b/doc/workflows/kubernetes/using_amazon_sagemaker_components.rst similarity index 100% rename from doc/kubernetes/using_amazon_sagemaker_components.rst rename to doc/workflows/kubernetes/using_amazon_sagemaker_components.rst From 8eebfe4288fcffaf3f7f11af866888fe4d76653b Mon Sep 17 00:00:00 2001 From: Lauren Yu <6631887+laurenyu@users.noreply.github.com> Date: Wed, 10 Jun 2020 13:36:27 -0700 Subject: [PATCH 06/18] change: make instance_type optional for prepare_container_def (#1567) This argument was used only for determining default framework image tags, and is not needed in many cases. --- src/sagemaker/chainer/model.py | 23 ++++++++--------- src/sagemaker/model.py | 8 +++--- src/sagemaker/multidatamodel.py | 2 +- src/sagemaker/mxnet/model.py | 7 ++++- src/sagemaker/pytorch/model.py | 7 ++++- src/sagemaker/sklearn/model.py | 28 +++++++------------- src/sagemaker/tensorflow/model.py | 7 ++++- src/sagemaker/tensorflow/serving.py | 7 ++++- src/sagemaker/xgboost/model.py | 33 +++++++++--------------- tests/unit/sagemaker/model/test_model.py | 18 ++++++++++--- tests/unit/test_chainer.py | 10 +++++++ tests/unit/test_mxnet.py | 10 +++++++ tests/unit/test_pytorch.py | 10 +++++++ tests/unit/test_tf_estimator.py | 10 +++++++ tests/unit/test_tfs.py | 10 +++++++ 15 files changed, 126 insertions(+), 64 deletions(-) diff --git a/src/sagemaker/chainer/model.py b/src/sagemaker/chainer/model.py index 8a742836a3..fc17be17a8 100644 --- a/src/sagemaker/chainer/model.py +++ b/src/sagemaker/chainer/model.py @@ -15,8 +15,6 @@ import logging -from sagemaker import fw_utils - import sagemaker from sagemaker.fw_utils import ( create_image_uri, @@ -126,7 +124,7 @@ def __init__( self.framework_version = framework_version or defaults.CHAINER_VERSION self.model_server_workers = model_server_workers - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition with framework configuration set in model environment variables. @@ -143,14 +141,14 @@ def prepare_container_def(self, instance_type, accelerator_type=None): """ deploy_image = self.image if not deploy_image: + if instance_type is None: + raise ValueError( + "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + ) + region_name = self.sagemaker_session.boto_session.region_name - deploy_image = create_image_uri( - region_name, - self.__framework_name__, - instance_type, - self.framework_version, - self.py_version, - accelerator_type=accelerator_type, + deploy_image = self.serving_image_uri( + region_name, instance_type, accelerator_type=accelerator_type ) deploy_key_prefix = model_code_key_prefix(self.key_prefix, self.name, deploy_image) @@ -162,7 +160,7 @@ def prepare_container_def(self, instance_type, accelerator_type=None): deploy_env[MODEL_SERVER_WORKERS_PARAM_NAME.upper()] = str(self.model_server_workers) return sagemaker.container_def(deploy_image, self.model_data, deploy_env) - def serving_image_uri(self, region_name, instance_type): + def serving_image_uri(self, region_name, instance_type, accelerator_type=None): """Create a URI for the serving image. Args: @@ -174,10 +172,11 @@ def serving_image_uri(self, region_name, instance_type): str: The appropriate image URI based on the given parameters. """ - return fw_utils.create_image_uri( + return create_image_uri( region_name, self.__framework_name__, instance_type, self.framework_version, self.py_version, + accelerator_type=accelerator_type, ) diff --git a/src/sagemaker/model.py b/src/sagemaker/model.py index b74f80fd31..e9d379f0b5 100644 --- a/src/sagemaker/model.py +++ b/src/sagemaker/model.py @@ -138,7 +138,7 @@ def _init_sagemaker_session_if_does_not_exist(self, instance_type): self.sagemaker_session = session.Session() def prepare_container_def( - self, instance_type, accelerator_type=None + self, instance_type=None, accelerator_type=None ): # pylint: disable=unused-argument """Return a dict created by ``sagemaker.container_def()`` for deploying this model to a specified instance type. @@ -166,7 +166,7 @@ def enable_network_isolation(self): """ return self._enable_network_isolation - def _create_sagemaker_model(self, instance_type, accelerator_type=None, tags=None): + def _create_sagemaker_model(self, instance_type=None, accelerator_type=None, tags=None): """Create a SageMaker Model Entity Args: @@ -807,9 +807,7 @@ def __init__( self.uploaded_code = None self.repacked_model_data = None - def prepare_container_def( - self, instance_type, accelerator_type=None - ): # pylint disable=unused-argument + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition with framework configuration set in model environment variables. diff --git a/src/sagemaker/multidatamodel.py b/src/sagemaker/multidatamodel.py index 89dbef5ebf..07edbdd987 100644 --- a/src/sagemaker/multidatamodel.py +++ b/src/sagemaker/multidatamodel.py @@ -111,7 +111,7 @@ def __init__( **kwargs ) - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition set with MultiModel mode, model data and other parameters from the model (if available). diff --git a/src/sagemaker/mxnet/model.py b/src/sagemaker/mxnet/model.py index 47691ff2b7..093a2f37bd 100644 --- a/src/sagemaker/mxnet/model.py +++ b/src/sagemaker/mxnet/model.py @@ -126,7 +126,7 @@ def __init__( self.framework_version = framework_version or defaults.MXNET_VERSION self.model_server_workers = model_server_workers - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition with framework configuration set in model environment variables. @@ -143,6 +143,11 @@ def prepare_container_def(self, instance_type, accelerator_type=None): """ deploy_image = self.image if not deploy_image: + if instance_type is None: + raise ValueError( + "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + ) + region_name = self.sagemaker_session.boto_session.region_name deploy_image = self.serving_image_uri( region_name, instance_type, accelerator_type=accelerator_type diff --git a/src/sagemaker/pytorch/model.py b/src/sagemaker/pytorch/model.py index bfcacf4975..38babdb7e2 100644 --- a/src/sagemaker/pytorch/model.py +++ b/src/sagemaker/pytorch/model.py @@ -127,7 +127,7 @@ def __init__( self.framework_version = framework_version or defaults.PYTORCH_VERSION self.model_server_workers = model_server_workers - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition with framework configuration set in model environment variables. @@ -144,6 +144,11 @@ def prepare_container_def(self, instance_type, accelerator_type=None): """ deploy_image = self.image if not deploy_image: + if instance_type is None: + raise ValueError( + "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + ) + region_name = self.sagemaker_session.boto_session.region_name deploy_image = self.serving_image_uri( region_name, instance_type, accelerator_type=accelerator_type diff --git a/src/sagemaker/sklearn/model.py b/src/sagemaker/sklearn/model.py index fce3d3e304..b06834df52 100644 --- a/src/sagemaker/sklearn/model.py +++ b/src/sagemaker/sklearn/model.py @@ -15,8 +15,6 @@ import logging -from sagemaker import fw_utils - import sagemaker from sagemaker.fw_utils import model_code_key_prefix, python_deprecation_warning from sagemaker.fw_registry import default_framework_uri @@ -118,16 +116,16 @@ def __init__( self.framework_version = framework_version self.model_server_workers = model_server_workers - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition with framework configuration set in model environment variables. Args: instance_type (str): The EC2 instance type to deploy this Model to. - For example, 'ml.p2.xlarge'. + This parameter is unused because Scikit-learn supports only CPU. accelerator_type (str): The Elastic Inference accelerator type to deploy to the instance for loading and making inferences to the - model. For example, 'ml.eia1.medium'. Note: accelerator types + model. This parameter is unused because accelerator types are not supported by SKLearnModel. Returns: @@ -139,9 +137,8 @@ def prepare_container_def(self, instance_type, accelerator_type=None): deploy_image = self.image if not deploy_image: - image_tag = "{}-{}-{}".format(self.framework_version, "cpu", self.py_version) - deploy_image = default_framework_uri( - self.__framework_name__, self.sagemaker_session.boto_region_name, image_tag + deploy_image = self.serving_image_uri( + self.sagemaker_session.boto_region_name, instance_type ) deploy_key_prefix = model_code_key_prefix(self.key_prefix, self.name, deploy_image) @@ -156,22 +153,17 @@ def prepare_container_def(self, instance_type, accelerator_type=None): ) return sagemaker.container_def(deploy_image, model_data_uri, deploy_env) - def serving_image_uri(self, region_name, instance_type): + def serving_image_uri(self, region_name, instance_type): # pylint: disable=unused-argument """Create a URI for the serving image. Args: region_name (str): AWS region where the image is uploaded. - instance_type (str): SageMaker instance type. Used to determine device type - (cpu/gpu/family-specific optimized). + instance_type (str): SageMaker instance type. This parameter is unused because + Scikit-learn supports only CPU. Returns: str: The appropriate image URI based on the given parameters. """ - return fw_utils.create_image_uri( - region_name, - self.__framework_name__, - instance_type, - self.framework_version, - self.py_version, - ) + image_tag = "{}-{}-{}".format(self.framework_version, "cpu", self.py_version) + return default_framework_uri(self.__framework_name__, region_name, image_tag) diff --git a/src/sagemaker/tensorflow/model.py b/src/sagemaker/tensorflow/model.py index 540e0dfe42..73c7830151 100644 --- a/src/sagemaker/tensorflow/model.py +++ b/src/sagemaker/tensorflow/model.py @@ -124,7 +124,7 @@ def __init__( self.framework_version = framework_version or defaults.TF_VERSION self.model_server_workers = model_server_workers - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition with framework configuration set in model environment variables. @@ -143,6 +143,11 @@ def prepare_container_def(self, instance_type, accelerator_type=None): """ deploy_image = self.image if not deploy_image: + if instance_type is None: + raise ValueError( + "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + ) + region_name = self.sagemaker_session.boto_region_name deploy_image = self.serving_image_uri( region_name, instance_type, accelerator_type=accelerator_type diff --git a/src/sagemaker/tensorflow/serving.py b/src/sagemaker/tensorflow/serving.py index 05fcae6428..ba7d16119b 100644 --- a/src/sagemaker/tensorflow/serving.py +++ b/src/sagemaker/tensorflow/serving.py @@ -215,12 +215,17 @@ def _eia_supported(self): """Return true if TF version is EIA enabled""" return [int(s) for s in self._framework_version.split(".")][:2] <= self.LATEST_EIA_VERSION - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """ Args: instance_type: accelerator_type: """ + if self.image is None and instance_type is None: + raise ValueError( + "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + ) + image = self._get_image_uri(instance_type, accelerator_type) env = self._get_container_env() diff --git a/src/sagemaker/xgboost/model.py b/src/sagemaker/xgboost/model.py index aef85437fd..7d73df0f10 100644 --- a/src/sagemaker/xgboost/model.py +++ b/src/sagemaker/xgboost/model.py @@ -15,8 +15,6 @@ import logging -from sagemaker import fw_utils - import sagemaker from sagemaker.fw_utils import model_code_key_prefix from sagemaker.fw_registry import default_framework_uri @@ -107,26 +105,24 @@ def __init__( self.framework_version = framework_version self.model_server_workers = model_server_workers - def prepare_container_def(self, instance_type, accelerator_type=None): + def prepare_container_def(self, instance_type=None, accelerator_type=None): """Return a container definition with framework configuration set in model environment variables. Args: - instance_type (str): The EC2 instance type to deploy this Model to. For example, - 'ml.m5.xlarge'. + instance_type (str): The EC2 instance type to deploy this Model to. + This parameter is unused because XGBoost supports only CPU. accelerator_type (str): The Elastic Inference accelerator type to deploy to the - instance for loading and making inferences to the model. For example, - 'ml.eia1.medium'. - Note: accelerator types are not supported by XGBoostModel. + instance for loading and making inferences to the model. This parameter is + unused because accelerator types are not supported by XGBoostModel. Returns: dict[str, str]: A container definition object usable with the CreateModel API. """ deploy_image = self.image if not deploy_image: - image_tag = "{}-{}-{}".format(self.framework_version, "cpu", self.py_version) - deploy_image = default_framework_uri( - self.__framework_name__, self.sagemaker_session.boto_region_name, image_tag + deploy_image = self.serving_image_uri( + self.sagemaker_session.boto_region_name, instance_type ) deploy_key_prefix = model_code_key_prefix(self.key_prefix, self.name, deploy_image) @@ -138,22 +134,17 @@ def prepare_container_def(self, instance_type, accelerator_type=None): deploy_env[MODEL_SERVER_WORKERS_PARAM_NAME.upper()] = str(self.model_server_workers) return sagemaker.container_def(deploy_image, self.model_data, deploy_env) - def serving_image_uri(self, region_name, instance_type): + def serving_image_uri(self, region_name, instance_type): # pylint: disable=unused-argument """Create a URI for the serving image. Args: region_name (str): AWS region where the image is uploaded. - instance_type (str): SageMaker instance type. Used to determine device type - (cpu/gpu/family-specific optimized). + instance_type (str): SageMaker instance type. This parameter is unused because + XGBoost supports only CPU. Returns: str: The appropriate image URI based on the given parameters. """ - return fw_utils.create_image_uri( - region_name, - self.__framework_name__, - instance_type, - self.framework_version, - self.py_version, - ) + image_tag = "{}-{}-{}".format(self.framework_version, "cpu", self.py_version) + return default_framework_uri(self.__framework_name__, region_name, image_tag) diff --git a/tests/unit/sagemaker/model/test_model.py b/tests/unit/sagemaker/model/test_model.py index b5f9dec4a5..c9b21d816c 100644 --- a/tests/unit/sagemaker/model/test_model.py +++ b/tests/unit/sagemaker/model/test_model.py @@ -37,9 +37,12 @@ def test_prepare_container_def(): env = {"FOO": "BAR"} model = Model(MODEL_DATA, MODEL_IMAGE, env=env) + expected = {"Image": MODEL_IMAGE, "Environment": env, "ModelDataUrl": MODEL_DATA} + container_def = model.prepare_container_def(INSTANCE_TYPE, "ml.eia.medium") + assert expected == container_def - expected = {"Image": MODEL_IMAGE, "Environment": env, "ModelDataUrl": MODEL_DATA} + container_def = model.prepare_container_def() assert expected == container_def @@ -60,9 +63,9 @@ def test_create_sagemaker_model(name_from_image, prepare_container_def, sagemake prepare_container_def.return_value = container_def model = Model(MODEL_DATA, MODEL_IMAGE, sagemaker_session=sagemaker_session) - model._create_sagemaker_model(INSTANCE_TYPE) + model._create_sagemaker_model() - prepare_container_def.assert_called_with(INSTANCE_TYPE, accelerator_type=None) + prepare_container_def.assert_called_with(None, accelerator_type=None) name_from_image.assert_called_with(MODEL_IMAGE) sagemaker_session.create_model.assert_called_with( @@ -70,6 +73,15 @@ def test_create_sagemaker_model(name_from_image, prepare_container_def, sagemake ) +@patch("sagemaker.utils.name_from_image", Mock()) +@patch("sagemaker.model.Model.prepare_container_def") +def test_create_sagemaker_model_instance_type(prepare_container_def, sagemaker_session): + model = Model(MODEL_DATA, MODEL_IMAGE, sagemaker_session=sagemaker_session) + model._create_sagemaker_model(INSTANCE_TYPE) + + prepare_container_def.assert_called_with(INSTANCE_TYPE, accelerator_type=None) + + @patch("sagemaker.utils.name_from_image", Mock()) @patch("sagemaker.model.Model.prepare_container_def") def test_create_sagemaker_model_accelerator_type(prepare_container_def, sagemaker_session): diff --git a/tests/unit/test_chainer.py b/tests/unit/test_chainer.py index cdc063ce54..6d4f34fb5b 100644 --- a/tests/unit/test_chainer.py +++ b/tests/unit/test_chainer.py @@ -434,6 +434,16 @@ def test_model_prepare_container_def_accelerator_error(sagemaker_session): model.prepare_container_def(INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE) +def test_model_prepare_container_def_no_instance_type_or_image(): + model = ChainerModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH) + + with pytest.raises(ValueError) as e: + model.prepare_container_def() + + expected_msg = "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + assert expected_msg in str(e) + + def test_train_image_default(sagemaker_session): chainer = Chainer( entry_point=SCRIPT_PATH, diff --git a/tests/unit/test_mxnet.py b/tests/unit/test_mxnet.py index e073462a3f..2b1aa6af65 100644 --- a/tests/unit/test_mxnet.py +++ b/tests/unit/test_mxnet.py @@ -491,6 +491,16 @@ def test_model_image_accelerator_mms_version(sagemaker_session): ) +def test_model_prepare_container_def_no_instance_type_or_image(): + model = MXNetModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH) + + with pytest.raises(ValueError) as e: + model.prepare_container_def() + + expected_msg = "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + assert expected_msg in str(e) + + def test_train_image_default(sagemaker_session): mx = MXNet( entry_point=SCRIPT_PATH, diff --git a/tests/unit/test_pytorch.py b/tests/unit/test_pytorch.py index 6851c7c4b5..e61ee962e2 100644 --- a/tests/unit/test_pytorch.py +++ b/tests/unit/test_pytorch.py @@ -368,6 +368,16 @@ def test_model_image_accelerator(sagemaker_session): ) +def test_model_prepare_container_def_no_instance_type_or_image(): + model = PyTorchModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH) + + with pytest.raises(ValueError) as e: + model.prepare_container_def() + + expected_msg = "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + assert expected_msg in str(e) + + def test_train_image_default(sagemaker_session): pytorch = PyTorch( entry_point=SCRIPT_PATH, diff --git a/tests/unit/test_tf_estimator.py b/tests/unit/test_tf_estimator.py index d00523c357..cbf678e089 100644 --- a/tests/unit/test_tf_estimator.py +++ b/tests/unit/test_tf_estimator.py @@ -575,6 +575,16 @@ def test_model_image_accelerator(sagemaker_session): assert container_def["Image"] == _get_full_cpu_image_uri_with_ei(defaults.TF_VERSION) +def test_model_prepare_container_def_no_instance_type_or_image(): + model = TensorFlowModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH) + + with pytest.raises(ValueError) as e: + model.prepare_container_def() + + expected_msg = "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + assert expected_msg in str(e) + + @patch("time.strftime", return_value=TIMESTAMP) @patch("time.time", return_value=TIME) @patch("subprocess.Popen") diff --git a/tests/unit/test_tfs.py b/tests/unit/test_tfs.py index b24a5a742f..ffe30f7587 100644 --- a/tests/unit/test_tfs.py +++ b/tests/unit/test_tfs.py @@ -245,6 +245,16 @@ def test_tfs_model_with_dependencies( ) +def test_model_prepare_container_def_no_instance_type_or_image(): + model = Model("s3://some/data.tar.gz", role=ROLE) + + with pytest.raises(ValueError) as e: + model.prepare_container_def() + + expected_msg = "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." + assert expected_msg in str(e) + + def test_estimator_deploy(sagemaker_session): container_log_level = '"logging.INFO"' source_dir = "s3://mybucket/source" From fad5c0df490caeb7c803eedb034d69230b92cf6f Mon Sep 17 00:00:00 2001 From: Aakash Pydi Date: Wed, 10 Jun 2020 18:40:08 -0500 Subject: [PATCH 07/18] feature: Support for multi variant endpoint invocation with target variant param (#1571) --- src/sagemaker/local/local_session.py | 4 + src/sagemaker/predictor.py | 13 +- tests/integ/test_multi_variant_endpoint.py | 309 +++++++++++++++++++++ tests/unit/test_predictor.py | 26 ++ 4 files changed, 349 insertions(+), 3 deletions(-) create mode 100644 tests/integ/test_multi_variant_endpoint.py diff --git a/src/sagemaker/local/local_session.py b/src/sagemaker/local/local_session.py index 1e80f6e1b4..6c23cea45a 100644 --- a/src/sagemaker/local/local_session.py +++ b/src/sagemaker/local/local_session.py @@ -343,6 +343,7 @@ def invoke_endpoint( Accept=None, CustomAttributes=None, TargetModel=None, + TargetVariant=None, ): """ @@ -370,6 +371,9 @@ def invoke_endpoint( if TargetModel is not None: headers["X-Amzn-SageMaker-Target-Model"] = TargetModel + if TargetVariant is not None: + headers["X-Amzn-SageMaker-Target-Variant"] = TargetVariant + r = self.http.request("POST", url, body=Body, preload_content=False, headers=headers) return {"Body": r, "ContentType": Accept} diff --git a/src/sagemaker/predictor.py b/src/sagemaker/predictor.py index 80da8a551c..0f1efb4e18 100644 --- a/src/sagemaker/predictor.py +++ b/src/sagemaker/predictor.py @@ -83,7 +83,7 @@ def __init__( self._endpoint_config_name = self._get_endpoint_config_name() self._model_names = self._get_model_names() - def predict(self, data, initial_args=None, target_model=None): + def predict(self, data, initial_args=None, target_model=None, target_variant=None): """Return the inference from the specified endpoint. Args: @@ -98,6 +98,9 @@ def predict(self, data, initial_args=None, target_model=None): target_model (str): S3 model artifact path to run an inference request on, in case of a multi model endpoint. Does not apply to endpoints hosting single model (Default: None) + target_variant (str): The name of the production variant to run an inference + request on (Default: None). Note that the ProductionVariant identifies the model + you want to host and the resources you want to deploy for hosting it. Returns: object: Inference for the given input. If a deserializer was specified when creating @@ -106,7 +109,7 @@ def predict(self, data, initial_args=None, target_model=None): as is. """ - request_args = self._create_request_args(data, initial_args, target_model) + request_args = self._create_request_args(data, initial_args, target_model, target_variant) response = self.sagemaker_session.sagemaker_runtime_client.invoke_endpoint(**request_args) return self._handle_response(response) @@ -123,12 +126,13 @@ def _handle_response(self, response): response_body.close() return data - def _create_request_args(self, data, initial_args=None, target_model=None): + def _create_request_args(self, data, initial_args=None, target_model=None, target_variant=None): """ Args: data: initial_args: target_model: + target_variant: """ args = dict(initial_args) if initial_args else {} @@ -144,6 +148,9 @@ def _create_request_args(self, data, initial_args=None, target_model=None): if target_model: args["TargetModel"] = target_model + if target_variant: + args["TargetVariant"] = target_variant + if self.serializer is not None: data = self.serializer(data) diff --git a/tests/integ/test_multi_variant_endpoint.py b/tests/integ/test_multi_variant_endpoint.py new file mode 100644 index 0000000000..7d1fab7645 --- /dev/null +++ b/tests/integ/test_multi_variant_endpoint.py @@ -0,0 +1,309 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from __future__ import absolute_import + +import json +import os +import math +import pytest +import scipy.stats as st + +from sagemaker.s3 import S3Uploader +from sagemaker.session import production_variant +from sagemaker.sparkml import SparkMLModel +from sagemaker.utils import sagemaker_timestamp +from sagemaker.content_types import CONTENT_TYPE_CSV +from sagemaker.utils import unique_name_from_base +from sagemaker.amazon.amazon_estimator import get_image_uri +from sagemaker.predictor import csv_serializer, RealTimePredictor + + +import tests.integ + + +ROLE = "SageMakerRole" +MODEL_NAME = "test-xgboost-model-{}".format(sagemaker_timestamp()) +ENDPOINT_NAME = unique_name_from_base("integ-test-multi-variant-endpoint") +DEFAULT_REGION = "us-west-2" +DEFAULT_INSTANCE_TYPE = "ml.m5.xlarge" +DEFAULT_INSTANCE_COUNT = 1 +XG_BOOST_MODEL_LOCAL_PATH = os.path.join(tests.integ.DATA_DIR, "xgboost_model", "xgb_model.tar.gz") + +TEST_VARIANT_1 = "Variant1" +TEST_VARIANT_1_WEIGHT = 0.3 + +TEST_VARIANT_2 = "Variant2" +TEST_VARIANT_2_WEIGHT = 0.7 + +VARIANT_TRAFFIC_SAMPLING_COUNT = 100 +DESIRED_CONFIDENCE_FOR_VARIANT_TRAFFIC_DISTRIBUTION = 0.999 + +TEST_CSV_DATA = "42,42,42,42,42,42,42" + +SPARK_ML_MODEL_LOCAL_PATH = os.path.join( + tests.integ.DATA_DIR, "sparkml_model", "mleap_model.tar.gz" +) +SPARK_ML_MODEL_ENDPOINT_NAME = unique_name_from_base("integ-test-target-variant-sparkml") +SPARK_ML_DEFAULT_VARIANT_NAME = ( + "AllTraffic" +) # default defined in src/sagemaker/session.py def production_variant +SPARK_ML_WRONG_VARIANT_NAME = "WRONG_VARIANT" +SPARK_ML_TEST_DATA = "1.0,C,38.0,71.5,1.0,female" +SPARK_ML_MODEL_SCHEMA = json.dumps( + { + "input": [ + {"name": "Pclass", "type": "float"}, + {"name": "Embarked", "type": "string"}, + {"name": "Age", "type": "float"}, + {"name": "Fare", "type": "float"}, + {"name": "SibSp", "type": "float"}, + {"name": "Sex", "type": "string"}, + ], + "output": {"name": "features", "struct": "vector", "type": "double"}, + } +) + + +@pytest.fixture(scope="module") +def multi_variant_endpoint(sagemaker_session): + """ + Sets up the multi variant endpoint before the integration tests run. + Cleans up the multi variant endpoint after the integration tests run. + """ + + with tests.integ.timeout.timeout_and_delete_endpoint_by_name( + endpoint_name=ENDPOINT_NAME, sagemaker_session=sagemaker_session, hours=2 + ): + + # Creating a model + bucket = sagemaker_session.default_bucket() + prefix = "sagemaker/DEMO-VariantTargeting" + model_url = S3Uploader.upload( + local_path=XG_BOOST_MODEL_LOCAL_PATH, + desired_s3_uri="s3://" + bucket + "/" + prefix, + session=sagemaker_session, + ) + + image_uri = get_image_uri(sagemaker_session.boto_session.region_name, "xgboost", "0.90-1") + + multi_variant_endpoint_model = sagemaker_session.create_model( + name=MODEL_NAME, + role=ROLE, + container_defs={"Image": image_uri, "ModelDataUrl": model_url}, + ) + + # Creating a multi variant endpoint + variant1 = production_variant( + model_name=MODEL_NAME, + instance_type=DEFAULT_INSTANCE_TYPE, + initial_instance_count=DEFAULT_INSTANCE_COUNT, + variant_name=TEST_VARIANT_1, + initial_weight=TEST_VARIANT_1_WEIGHT, + ) + variant2 = production_variant( + model_name=MODEL_NAME, + instance_type=DEFAULT_INSTANCE_TYPE, + initial_instance_count=DEFAULT_INSTANCE_COUNT, + variant_name=TEST_VARIANT_2, + initial_weight=TEST_VARIANT_2_WEIGHT, + ) + sagemaker_session.endpoint_from_production_variants( + name=ENDPOINT_NAME, production_variants=[variant1, variant2] + ) + + # Yield to run the integration tests + yield multi_variant_endpoint + + # Cleanup resources + sagemaker_session.delete_model(multi_variant_endpoint_model) + sagemaker_session.sagemaker_client.delete_endpoint_config(EndpointConfigName=ENDPOINT_NAME) + + # Validate resource cleanup + with pytest.raises(Exception) as exception: + sagemaker_session.sagemaker_client.describe_model( + ModelName=multi_variant_endpoint_model.name + ) + assert "Could not find model" in str(exception.value) + sagemaker_session.sagemaker_client.describe_endpoint_config(name=ENDPOINT_NAME) + assert "Could not find endpoint" in str(exception.value) + + +def test_target_variant_invocation(sagemaker_session, multi_variant_endpoint): + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=ENDPOINT_NAME, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_1, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_1 + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=ENDPOINT_NAME, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_2, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_2 + + +def test_predict_invocation_with_target_variant(sagemaker_session, multi_variant_endpoint): + predictor = RealTimePredictor( + endpoint=ENDPOINT_NAME, + sagemaker_session=sagemaker_session, + serializer=csv_serializer, + content_type=CONTENT_TYPE_CSV, + accept=CONTENT_TYPE_CSV, + ) + + # Validate that no exception is raised when the target_variant is specified. + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2) + + +def test_variant_traffic_distribution(sagemaker_session, multi_variant_endpoint): + variant_1_invocation_count = 0 + variant_2_invocation_count = 0 + + for i in range(0, VARIANT_TRAFFIC_SAMPLING_COUNT): + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=ENDPOINT_NAME, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + ) + if response["InvokedProductionVariant"] == TEST_VARIANT_1: + variant_1_invocation_count += 1 + elif response["InvokedProductionVariant"] == TEST_VARIANT_2: + variant_2_invocation_count += 1 + + assert variant_1_invocation_count + variant_2_invocation_count == VARIANT_TRAFFIC_SAMPLING_COUNT + + variant_1_invocation_percentage = float(variant_1_invocation_count) / float( + VARIANT_TRAFFIC_SAMPLING_COUNT + ) + variant_1_margin_of_error = _compute_and_retrieve_margin_of_error(TEST_VARIANT_1_WEIGHT) + assert variant_1_invocation_percentage < TEST_VARIANT_1_WEIGHT + variant_1_margin_of_error + assert variant_1_invocation_percentage > TEST_VARIANT_1_WEIGHT - variant_1_margin_of_error + + variant_2_invocation_percentage = float(variant_2_invocation_count) / float( + VARIANT_TRAFFIC_SAMPLING_COUNT + ) + variant_2_margin_of_error = _compute_and_retrieve_margin_of_error(TEST_VARIANT_2_WEIGHT) + assert variant_2_invocation_percentage < TEST_VARIANT_2_WEIGHT + variant_2_margin_of_error + assert variant_2_invocation_percentage > TEST_VARIANT_2_WEIGHT - variant_2_margin_of_error + + +def test_spark_ml_predict_invocation_with_target_variant(sagemaker_session): + model_data = sagemaker_session.upload_data( + path=SPARK_ML_MODEL_LOCAL_PATH, key_prefix="integ-test-data/sparkml/model" + ) + + with tests.integ.timeout.timeout_and_delete_endpoint_by_name( + SPARK_ML_MODEL_ENDPOINT_NAME, sagemaker_session + ): + spark_ml_model = SparkMLModel( + model_data=model_data, + role=ROLE, + sagemaker_session=sagemaker_session, + env={"SAGEMAKER_SPARKML_SCHEMA": SPARK_ML_MODEL_SCHEMA}, + ) + + predictor = spark_ml_model.deploy( + DEFAULT_INSTANCE_COUNT, + DEFAULT_INSTANCE_TYPE, + endpoint_name=SPARK_ML_MODEL_ENDPOINT_NAME, + ) + + # Validate that no exception is raised when the target_variant is specified. + predictor.predict(SPARK_ML_TEST_DATA, target_variant=SPARK_ML_DEFAULT_VARIANT_NAME) + + with pytest.raises(Exception) as exception_info: + predictor.predict(SPARK_ML_TEST_DATA, target_variant=SPARK_ML_WRONG_VARIANT_NAME) + + assert "ValidationError" in str(exception_info.value) + assert SPARK_ML_WRONG_VARIANT_NAME in str(exception_info.value) + + # cleanup resources + spark_ml_model.delete_model() + sagemaker_session.sagemaker_client.delete_endpoint_config( + EndpointConfigName=SPARK_ML_MODEL_ENDPOINT_NAME + ) + + # Validate resource cleanup + with pytest.raises(Exception) as exception: + sagemaker_session.sagemaker_client.describe_model(ModelName=spark_ml_model.name) + assert "Could not find model" in str(exception.value) + sagemaker_session.sagemaker_client.describe_endpoint_config( + name=SPARK_ML_MODEL_ENDPOINT_NAME + ) + assert "Could not find endpoint" in str(exception.value) + + +@pytest.mark.local_mode +def test_target_variant_invocation_local_mode(sagemaker_session, multi_variant_endpoint): + + if sagemaker_session._region_name is None: + sagemaker_session._region_name = DEFAULT_REGION + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=ENDPOINT_NAME, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_1, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_1 + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=ENDPOINT_NAME, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_2, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_2 + + +@pytest.mark.local_mode +def test_predict_invocation_with_target_variant_local_mode( + sagemaker_session, multi_variant_endpoint +): + + if sagemaker_session._region_name is None: + sagemaker_session._region_name = DEFAULT_REGION + + predictor = RealTimePredictor( + endpoint=ENDPOINT_NAME, + sagemaker_session=sagemaker_session, + serializer=csv_serializer, + content_type=CONTENT_TYPE_CSV, + accept=CONTENT_TYPE_CSV, + ) + + # Validate that no exception is raised when the target_variant is specified. + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2) + + +def _compute_and_retrieve_margin_of_error(variant_weight): + """ + Computes the margin of error using the Wald method for computing the confidence + intervals of a binomial distribution. + """ + z_value = st.norm.ppf(DESIRED_CONFIDENCE_FOR_VARIANT_TRAFFIC_DISTRIBUTION) + margin_of_error = (variant_weight * (1 - variant_weight)) / VARIANT_TRAFFIC_SAMPLING_COUNT + margin_of_error = z_value * math.sqrt(margin_of_error) + return margin_of_error diff --git a/tests/unit/test_predictor.py b/tests/unit/test_predictor.py index 9c7feb0a83..b74f36e2b3 100644 --- a/tests/unit/test_predictor.py +++ b/tests/unit/test_predictor.py @@ -346,6 +346,7 @@ def test_numpy_deser_from_npy_object_array(): CSV_CONTENT_TYPE = "text/csv" RETURN_VALUE = 0 CSV_RETURN_VALUE = "1,2,3\r\n" +PRODUCTION_VARIANT_1 = "PRODUCTION_VARIANT_1" ENDPOINT_DESC = {"EndpointConfigName": ENDPOINT} @@ -407,6 +408,31 @@ def test_predict_call_with_headers(): assert result == RETURN_VALUE +def test_predict_call_with_target_variant(): + sagemaker_session = empty_sagemaker_session() + predictor = RealTimePredictor( + ENDPOINT, sagemaker_session, content_type=DEFAULT_CONTENT_TYPE, accept=DEFAULT_CONTENT_TYPE + ) + + data = "untouched" + result = predictor.predict(data, target_variant=PRODUCTION_VARIANT_1) + + assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called + + expected_request_args = { + "Accept": DEFAULT_CONTENT_TYPE, + "Body": data, + "ContentType": DEFAULT_CONTENT_TYPE, + "EndpointName": ENDPOINT, + "TargetVariant": PRODUCTION_VARIANT_1, + } + + call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args + assert kwargs == expected_request_args + + assert result == RETURN_VALUE + + def test_multi_model_predict_call_with_headers(): sagemaker_session = empty_sagemaker_session() predictor = RealTimePredictor( From fb5d7a32d851be69816d39cb2cac0aa8d273b2c4 Mon Sep 17 00:00:00 2001 From: Chuyang Date: Thu, 11 Jun 2020 08:48:12 -0700 Subject: [PATCH 08/18] Revert "feature: Support for multi variant endpoint invocation with target variant param (#1571)" (#1574) This reverts commit b481c36be38d08a72d23b15251d465331a1b7d86. --- src/sagemaker/local/local_session.py | 4 - src/sagemaker/predictor.py | 13 +- tests/integ/test_multi_variant_endpoint.py | 309 --------------------- tests/unit/test_predictor.py | 26 -- 4 files changed, 3 insertions(+), 349 deletions(-) delete mode 100644 tests/integ/test_multi_variant_endpoint.py diff --git a/src/sagemaker/local/local_session.py b/src/sagemaker/local/local_session.py index 6c23cea45a..1e80f6e1b4 100644 --- a/src/sagemaker/local/local_session.py +++ b/src/sagemaker/local/local_session.py @@ -343,7 +343,6 @@ def invoke_endpoint( Accept=None, CustomAttributes=None, TargetModel=None, - TargetVariant=None, ): """ @@ -371,9 +370,6 @@ def invoke_endpoint( if TargetModel is not None: headers["X-Amzn-SageMaker-Target-Model"] = TargetModel - if TargetVariant is not None: - headers["X-Amzn-SageMaker-Target-Variant"] = TargetVariant - r = self.http.request("POST", url, body=Body, preload_content=False, headers=headers) return {"Body": r, "ContentType": Accept} diff --git a/src/sagemaker/predictor.py b/src/sagemaker/predictor.py index 0f1efb4e18..80da8a551c 100644 --- a/src/sagemaker/predictor.py +++ b/src/sagemaker/predictor.py @@ -83,7 +83,7 @@ def __init__( self._endpoint_config_name = self._get_endpoint_config_name() self._model_names = self._get_model_names() - def predict(self, data, initial_args=None, target_model=None, target_variant=None): + def predict(self, data, initial_args=None, target_model=None): """Return the inference from the specified endpoint. Args: @@ -98,9 +98,6 @@ def predict(self, data, initial_args=None, target_model=None, target_variant=Non target_model (str): S3 model artifact path to run an inference request on, in case of a multi model endpoint. Does not apply to endpoints hosting single model (Default: None) - target_variant (str): The name of the production variant to run an inference - request on (Default: None). Note that the ProductionVariant identifies the model - you want to host and the resources you want to deploy for hosting it. Returns: object: Inference for the given input. If a deserializer was specified when creating @@ -109,7 +106,7 @@ def predict(self, data, initial_args=None, target_model=None, target_variant=Non as is. """ - request_args = self._create_request_args(data, initial_args, target_model, target_variant) + request_args = self._create_request_args(data, initial_args, target_model) response = self.sagemaker_session.sagemaker_runtime_client.invoke_endpoint(**request_args) return self._handle_response(response) @@ -126,13 +123,12 @@ def _handle_response(self, response): response_body.close() return data - def _create_request_args(self, data, initial_args=None, target_model=None, target_variant=None): + def _create_request_args(self, data, initial_args=None, target_model=None): """ Args: data: initial_args: target_model: - target_variant: """ args = dict(initial_args) if initial_args else {} @@ -148,9 +144,6 @@ def _create_request_args(self, data, initial_args=None, target_model=None, targe if target_model: args["TargetModel"] = target_model - if target_variant: - args["TargetVariant"] = target_variant - if self.serializer is not None: data = self.serializer(data) diff --git a/tests/integ/test_multi_variant_endpoint.py b/tests/integ/test_multi_variant_endpoint.py deleted file mode 100644 index 7d1fab7645..0000000000 --- a/tests/integ/test_multi_variant_endpoint.py +++ /dev/null @@ -1,309 +0,0 @@ -# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -from __future__ import absolute_import - -import json -import os -import math -import pytest -import scipy.stats as st - -from sagemaker.s3 import S3Uploader -from sagemaker.session import production_variant -from sagemaker.sparkml import SparkMLModel -from sagemaker.utils import sagemaker_timestamp -from sagemaker.content_types import CONTENT_TYPE_CSV -from sagemaker.utils import unique_name_from_base -from sagemaker.amazon.amazon_estimator import get_image_uri -from sagemaker.predictor import csv_serializer, RealTimePredictor - - -import tests.integ - - -ROLE = "SageMakerRole" -MODEL_NAME = "test-xgboost-model-{}".format(sagemaker_timestamp()) -ENDPOINT_NAME = unique_name_from_base("integ-test-multi-variant-endpoint") -DEFAULT_REGION = "us-west-2" -DEFAULT_INSTANCE_TYPE = "ml.m5.xlarge" -DEFAULT_INSTANCE_COUNT = 1 -XG_BOOST_MODEL_LOCAL_PATH = os.path.join(tests.integ.DATA_DIR, "xgboost_model", "xgb_model.tar.gz") - -TEST_VARIANT_1 = "Variant1" -TEST_VARIANT_1_WEIGHT = 0.3 - -TEST_VARIANT_2 = "Variant2" -TEST_VARIANT_2_WEIGHT = 0.7 - -VARIANT_TRAFFIC_SAMPLING_COUNT = 100 -DESIRED_CONFIDENCE_FOR_VARIANT_TRAFFIC_DISTRIBUTION = 0.999 - -TEST_CSV_DATA = "42,42,42,42,42,42,42" - -SPARK_ML_MODEL_LOCAL_PATH = os.path.join( - tests.integ.DATA_DIR, "sparkml_model", "mleap_model.tar.gz" -) -SPARK_ML_MODEL_ENDPOINT_NAME = unique_name_from_base("integ-test-target-variant-sparkml") -SPARK_ML_DEFAULT_VARIANT_NAME = ( - "AllTraffic" -) # default defined in src/sagemaker/session.py def production_variant -SPARK_ML_WRONG_VARIANT_NAME = "WRONG_VARIANT" -SPARK_ML_TEST_DATA = "1.0,C,38.0,71.5,1.0,female" -SPARK_ML_MODEL_SCHEMA = json.dumps( - { - "input": [ - {"name": "Pclass", "type": "float"}, - {"name": "Embarked", "type": "string"}, - {"name": "Age", "type": "float"}, - {"name": "Fare", "type": "float"}, - {"name": "SibSp", "type": "float"}, - {"name": "Sex", "type": "string"}, - ], - "output": {"name": "features", "struct": "vector", "type": "double"}, - } -) - - -@pytest.fixture(scope="module") -def multi_variant_endpoint(sagemaker_session): - """ - Sets up the multi variant endpoint before the integration tests run. - Cleans up the multi variant endpoint after the integration tests run. - """ - - with tests.integ.timeout.timeout_and_delete_endpoint_by_name( - endpoint_name=ENDPOINT_NAME, sagemaker_session=sagemaker_session, hours=2 - ): - - # Creating a model - bucket = sagemaker_session.default_bucket() - prefix = "sagemaker/DEMO-VariantTargeting" - model_url = S3Uploader.upload( - local_path=XG_BOOST_MODEL_LOCAL_PATH, - desired_s3_uri="s3://" + bucket + "/" + prefix, - session=sagemaker_session, - ) - - image_uri = get_image_uri(sagemaker_session.boto_session.region_name, "xgboost", "0.90-1") - - multi_variant_endpoint_model = sagemaker_session.create_model( - name=MODEL_NAME, - role=ROLE, - container_defs={"Image": image_uri, "ModelDataUrl": model_url}, - ) - - # Creating a multi variant endpoint - variant1 = production_variant( - model_name=MODEL_NAME, - instance_type=DEFAULT_INSTANCE_TYPE, - initial_instance_count=DEFAULT_INSTANCE_COUNT, - variant_name=TEST_VARIANT_1, - initial_weight=TEST_VARIANT_1_WEIGHT, - ) - variant2 = production_variant( - model_name=MODEL_NAME, - instance_type=DEFAULT_INSTANCE_TYPE, - initial_instance_count=DEFAULT_INSTANCE_COUNT, - variant_name=TEST_VARIANT_2, - initial_weight=TEST_VARIANT_2_WEIGHT, - ) - sagemaker_session.endpoint_from_production_variants( - name=ENDPOINT_NAME, production_variants=[variant1, variant2] - ) - - # Yield to run the integration tests - yield multi_variant_endpoint - - # Cleanup resources - sagemaker_session.delete_model(multi_variant_endpoint_model) - sagemaker_session.sagemaker_client.delete_endpoint_config(EndpointConfigName=ENDPOINT_NAME) - - # Validate resource cleanup - with pytest.raises(Exception) as exception: - sagemaker_session.sagemaker_client.describe_model( - ModelName=multi_variant_endpoint_model.name - ) - assert "Could not find model" in str(exception.value) - sagemaker_session.sagemaker_client.describe_endpoint_config(name=ENDPOINT_NAME) - assert "Could not find endpoint" in str(exception.value) - - -def test_target_variant_invocation(sagemaker_session, multi_variant_endpoint): - - response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( - EndpointName=ENDPOINT_NAME, - Body=TEST_CSV_DATA, - ContentType=CONTENT_TYPE_CSV, - Accept=CONTENT_TYPE_CSV, - TargetVariant=TEST_VARIANT_1, - ) - assert response["InvokedProductionVariant"] == TEST_VARIANT_1 - - response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( - EndpointName=ENDPOINT_NAME, - Body=TEST_CSV_DATA, - ContentType=CONTENT_TYPE_CSV, - Accept=CONTENT_TYPE_CSV, - TargetVariant=TEST_VARIANT_2, - ) - assert response["InvokedProductionVariant"] == TEST_VARIANT_2 - - -def test_predict_invocation_with_target_variant(sagemaker_session, multi_variant_endpoint): - predictor = RealTimePredictor( - endpoint=ENDPOINT_NAME, - sagemaker_session=sagemaker_session, - serializer=csv_serializer, - content_type=CONTENT_TYPE_CSV, - accept=CONTENT_TYPE_CSV, - ) - - # Validate that no exception is raised when the target_variant is specified. - predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) - predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2) - - -def test_variant_traffic_distribution(sagemaker_session, multi_variant_endpoint): - variant_1_invocation_count = 0 - variant_2_invocation_count = 0 - - for i in range(0, VARIANT_TRAFFIC_SAMPLING_COUNT): - response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( - EndpointName=ENDPOINT_NAME, - Body=TEST_CSV_DATA, - ContentType=CONTENT_TYPE_CSV, - Accept=CONTENT_TYPE_CSV, - ) - if response["InvokedProductionVariant"] == TEST_VARIANT_1: - variant_1_invocation_count += 1 - elif response["InvokedProductionVariant"] == TEST_VARIANT_2: - variant_2_invocation_count += 1 - - assert variant_1_invocation_count + variant_2_invocation_count == VARIANT_TRAFFIC_SAMPLING_COUNT - - variant_1_invocation_percentage = float(variant_1_invocation_count) / float( - VARIANT_TRAFFIC_SAMPLING_COUNT - ) - variant_1_margin_of_error = _compute_and_retrieve_margin_of_error(TEST_VARIANT_1_WEIGHT) - assert variant_1_invocation_percentage < TEST_VARIANT_1_WEIGHT + variant_1_margin_of_error - assert variant_1_invocation_percentage > TEST_VARIANT_1_WEIGHT - variant_1_margin_of_error - - variant_2_invocation_percentage = float(variant_2_invocation_count) / float( - VARIANT_TRAFFIC_SAMPLING_COUNT - ) - variant_2_margin_of_error = _compute_and_retrieve_margin_of_error(TEST_VARIANT_2_WEIGHT) - assert variant_2_invocation_percentage < TEST_VARIANT_2_WEIGHT + variant_2_margin_of_error - assert variant_2_invocation_percentage > TEST_VARIANT_2_WEIGHT - variant_2_margin_of_error - - -def test_spark_ml_predict_invocation_with_target_variant(sagemaker_session): - model_data = sagemaker_session.upload_data( - path=SPARK_ML_MODEL_LOCAL_PATH, key_prefix="integ-test-data/sparkml/model" - ) - - with tests.integ.timeout.timeout_and_delete_endpoint_by_name( - SPARK_ML_MODEL_ENDPOINT_NAME, sagemaker_session - ): - spark_ml_model = SparkMLModel( - model_data=model_data, - role=ROLE, - sagemaker_session=sagemaker_session, - env={"SAGEMAKER_SPARKML_SCHEMA": SPARK_ML_MODEL_SCHEMA}, - ) - - predictor = spark_ml_model.deploy( - DEFAULT_INSTANCE_COUNT, - DEFAULT_INSTANCE_TYPE, - endpoint_name=SPARK_ML_MODEL_ENDPOINT_NAME, - ) - - # Validate that no exception is raised when the target_variant is specified. - predictor.predict(SPARK_ML_TEST_DATA, target_variant=SPARK_ML_DEFAULT_VARIANT_NAME) - - with pytest.raises(Exception) as exception_info: - predictor.predict(SPARK_ML_TEST_DATA, target_variant=SPARK_ML_WRONG_VARIANT_NAME) - - assert "ValidationError" in str(exception_info.value) - assert SPARK_ML_WRONG_VARIANT_NAME in str(exception_info.value) - - # cleanup resources - spark_ml_model.delete_model() - sagemaker_session.sagemaker_client.delete_endpoint_config( - EndpointConfigName=SPARK_ML_MODEL_ENDPOINT_NAME - ) - - # Validate resource cleanup - with pytest.raises(Exception) as exception: - sagemaker_session.sagemaker_client.describe_model(ModelName=spark_ml_model.name) - assert "Could not find model" in str(exception.value) - sagemaker_session.sagemaker_client.describe_endpoint_config( - name=SPARK_ML_MODEL_ENDPOINT_NAME - ) - assert "Could not find endpoint" in str(exception.value) - - -@pytest.mark.local_mode -def test_target_variant_invocation_local_mode(sagemaker_session, multi_variant_endpoint): - - if sagemaker_session._region_name is None: - sagemaker_session._region_name = DEFAULT_REGION - - response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( - EndpointName=ENDPOINT_NAME, - Body=TEST_CSV_DATA, - ContentType=CONTENT_TYPE_CSV, - Accept=CONTENT_TYPE_CSV, - TargetVariant=TEST_VARIANT_1, - ) - assert response["InvokedProductionVariant"] == TEST_VARIANT_1 - - response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( - EndpointName=ENDPOINT_NAME, - Body=TEST_CSV_DATA, - ContentType=CONTENT_TYPE_CSV, - Accept=CONTENT_TYPE_CSV, - TargetVariant=TEST_VARIANT_2, - ) - assert response["InvokedProductionVariant"] == TEST_VARIANT_2 - - -@pytest.mark.local_mode -def test_predict_invocation_with_target_variant_local_mode( - sagemaker_session, multi_variant_endpoint -): - - if sagemaker_session._region_name is None: - sagemaker_session._region_name = DEFAULT_REGION - - predictor = RealTimePredictor( - endpoint=ENDPOINT_NAME, - sagemaker_session=sagemaker_session, - serializer=csv_serializer, - content_type=CONTENT_TYPE_CSV, - accept=CONTENT_TYPE_CSV, - ) - - # Validate that no exception is raised when the target_variant is specified. - predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) - predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2) - - -def _compute_and_retrieve_margin_of_error(variant_weight): - """ - Computes the margin of error using the Wald method for computing the confidence - intervals of a binomial distribution. - """ - z_value = st.norm.ppf(DESIRED_CONFIDENCE_FOR_VARIANT_TRAFFIC_DISTRIBUTION) - margin_of_error = (variant_weight * (1 - variant_weight)) / VARIANT_TRAFFIC_SAMPLING_COUNT - margin_of_error = z_value * math.sqrt(margin_of_error) - return margin_of_error diff --git a/tests/unit/test_predictor.py b/tests/unit/test_predictor.py index b74f36e2b3..9c7feb0a83 100644 --- a/tests/unit/test_predictor.py +++ b/tests/unit/test_predictor.py @@ -346,7 +346,6 @@ def test_numpy_deser_from_npy_object_array(): CSV_CONTENT_TYPE = "text/csv" RETURN_VALUE = 0 CSV_RETURN_VALUE = "1,2,3\r\n" -PRODUCTION_VARIANT_1 = "PRODUCTION_VARIANT_1" ENDPOINT_DESC = {"EndpointConfigName": ENDPOINT} @@ -408,31 +407,6 @@ def test_predict_call_with_headers(): assert result == RETURN_VALUE -def test_predict_call_with_target_variant(): - sagemaker_session = empty_sagemaker_session() - predictor = RealTimePredictor( - ENDPOINT, sagemaker_session, content_type=DEFAULT_CONTENT_TYPE, accept=DEFAULT_CONTENT_TYPE - ) - - data = "untouched" - result = predictor.predict(data, target_variant=PRODUCTION_VARIANT_1) - - assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called - - expected_request_args = { - "Accept": DEFAULT_CONTENT_TYPE, - "Body": data, - "ContentType": DEFAULT_CONTENT_TYPE, - "EndpointName": ENDPOINT, - "TargetVariant": PRODUCTION_VARIANT_1, - } - - call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args - assert kwargs == expected_request_args - - assert result == RETURN_VALUE - - def test_multi_model_predict_call_with_headers(): sagemaker_session = empty_sagemaker_session() predictor = RealTimePredictor( From 092097200e6f5a39188c5606a9dc4813e8886b93 Mon Sep 17 00:00:00 2001 From: Theofilos Papapanagiotou Date: Thu, 11 Jun 2020 18:52:12 +0200 Subject: [PATCH 09/18] doc: fix typo in MXNet documentation (#1575) Co-authored-by: Chuyang --- doc/frameworks/mxnet/using_mxnet.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/frameworks/mxnet/using_mxnet.rst b/doc/frameworks/mxnet/using_mxnet.rst index c91522a219..abf87094f5 100644 --- a/doc/frameworks/mxnet/using_mxnet.rst +++ b/doc/frameworks/mxnet/using_mxnet.rst @@ -176,7 +176,7 @@ The default serialization system generates three files: - ``model-symbol.json``: The MXNet ``Module`` ``Symbol`` serialization, produced by invoking ``save`` on the ``symbol`` property of the ``Module`` being saved. -- ``modle.params``: The MXNet ``Module`` parameters, produced by +- ``model.params``: The MXNet ``Module`` parameters, produced by invoking ``save_params`` on the ``Module`` being saved. You can provide your own save function. This is useful if you are not working with the ``Module`` API or you need special processing. From 77b7c7fd8eebad29962eb37e5fc24b10e8fcddf4 Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 11 Jun 2020 17:18:37 +0000 Subject: [PATCH 10/18] prepare release v1.62.0 --- CHANGELOG.md | 16 ++++++++++++++++ VERSION | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f9bb21b0..f6070d3862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## v1.62.0 (2020-06-11) + +### Features + + * Support for multi variant endpoint invocation with target variant param + +### Bug Fixes and Other Changes + + * Revert "feature: Support for multi variant endpoint invocation with target variant param (#1571)" + * make instance_type optional for prepare_container_def + * docs: workflows navigation + +### Documentation Changes + + * fix typo in MXNet documentation + ## v1.61.0 (2020-06-09) ### Features diff --git a/VERSION b/VERSION index 16cd31157e..76d0536205 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.61.1.dev0 +1.62.0 From 47b668a2dd26ebe6ae86d61737d3b41adf8f15a7 Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 11 Jun 2020 17:49:34 +0000 Subject: [PATCH 11/18] update development version to v1.62.1.dev0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 76d0536205..69f7b1f2a8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.62.0 +1.62.1.dev0 From faa4993a2e7101811dc9fded7fe22f6f6cf19cd9 Mon Sep 17 00:00:00 2001 From: Chuyang Date: Thu, 11 Jun 2020 15:59:15 -0700 Subject: [PATCH 12/18] doc: improve docstring and remove unavailable links (#1572) * doc: improve docstring and remove unavailable links * remove unavailble links Co-authored-by: Chuyang Deng --- src/sagemaker/chainer/estimator.py | 5 ++--- src/sagemaker/pytorch/estimator.py | 5 ++--- src/sagemaker/s3.py | 5 +++-- src/sagemaker/sklearn/estimator.py | 3 +-- src/sagemaker/tensorflow/estimator.py | 4 +--- src/sagemaker/xgboost/estimator.py | 3 +-- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/sagemaker/chainer/estimator.py b/src/sagemaker/chainer/estimator.py index 0709087d64..19016447f8 100644 --- a/src/sagemaker/chainer/estimator.py +++ b/src/sagemaker/chainer/estimator.py @@ -105,9 +105,8 @@ def __init__( py_version (str): Python version you want to use for executing your model training code (default: 'py2'). One of 'py2' or 'py3'. framework_version (str): Chainer version you want to use for - executing your model training code. List of supported versions - https://github.com/aws/sagemaker-python-sdk#chainer-sagemaker-estimators. - If not specified, this will default to 4.1. + executing your model training code. If not specified, this will + default to 4.1. image_name (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and diff --git a/src/sagemaker/pytorch/estimator.py b/src/sagemaker/pytorch/estimator.py index 4a388acbee..0ba66b00d8 100644 --- a/src/sagemaker/pytorch/estimator.py +++ b/src/sagemaker/pytorch/estimator.py @@ -83,9 +83,8 @@ def __init__( py_version (str): Python version you want to use for executing your model training code (default: 'py3'). One of 'py2' or 'py3'. framework_version (str): PyTorch version you want to use for - executing your model training code. List of supported versions - https://github.com/aws/sagemaker-python-sdk#pytorch-sagemaker-estimators. - If not specified, this will default to 0.4. + executing your model training code. If not specified, this will default + to 0.4. image_name (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and diff --git a/src/sagemaker/s3.py b/src/sagemaker/s3.py index b710fdbe71..3c92aee946 100644 --- a/src/sagemaker/s3.py +++ b/src/sagemaker/s3.py @@ -57,8 +57,9 @@ def upload(local_path, desired_s3_uri, kms_key=None, session=None): """Static method that uploads a given file or directory to S3. Args: - local_path (str): A local path to a file or directory. - desired_s3_uri (str): The desired S3 uri to upload to. + local_path (str): Path (absolute or relative) of local file or directory to upload. + desired_s3_uri (str): The desired S3 location to upload to. It is the prefix to + which the local filename will be added. kms_key (str): The KMS key to use to encrypt the files. session (sagemaker.session.Session): Session object which manages interactions with Amazon SageMaker APIs and any other diff --git a/src/sagemaker/sklearn/estimator.py b/src/sagemaker/sklearn/estimator.py index e0fc4f76b9..4e6a958d6b 100644 --- a/src/sagemaker/sklearn/estimator.py +++ b/src/sagemaker/sklearn/estimator.py @@ -68,8 +68,7 @@ def __init__( If ``source_dir`` is specified, then ``entry_point`` must point to a file located at the root of ``source_dir``. framework_version (str): Scikit-learn version you want to use for - executing your model training code. List of supported versions - https://github.com/aws/sagemaker-python-sdk#sklearn-sagemaker-estimators + executing your model training code. source_dir (str): Path (absolute, relative or an S3 URI) to a directory with any other training source code dependencies aside from the entry point file (default: None). If ``source_dir`` is an S3 URI, it must diff --git a/src/sagemaker/tensorflow/estimator.py b/src/sagemaker/tensorflow/estimator.py index 79ce217676..c29daf9f69 100644 --- a/src/sagemaker/tensorflow/estimator.py +++ b/src/sagemaker/tensorflow/estimator.py @@ -237,9 +237,7 @@ def __init__( py_version (str): Python version you want to use for executing your model training code (default: 'py2'). framework_version (str): TensorFlow version you want to use for executing your model - training code. List of supported versions - https://github.com/aws/sagemaker-python-sdk#tensorflow-sagemaker-estimators. - If not specified, this will default to 1.11. + training code. If not specified, this will default to 1.11. model_dir (str): S3 location where the checkpoint data and models can be exported to during training (default: None). It will be passed in the training script as one of the command line arguments. If not specified, one is provided based on diff --git a/src/sagemaker/xgboost/estimator.py b/src/sagemaker/xgboost/estimator.py index 134235d829..c6b0b5a95a 100644 --- a/src/sagemaker/xgboost/estimator.py +++ b/src/sagemaker/xgboost/estimator.py @@ -73,8 +73,7 @@ def __init__( be executed as the entry point to training. If ``source_dir`` is specified, then ``entry_point`` must point to a file located at the root of ``source_dir``. framework_version (str): XGBoost version you want to use for executing your model - training code. List of supported versions - https://github.com/aws/sagemaker-python-sdk#xgboost-sagemaker-estimators + training code. source_dir (str): Path (absolute, relative or an S3 URI) to a directory with any other training source code dependencies aside from the entry point file (default: None). If ``source_dir`` is an S3 URI, it must From 47df02188256e868ed411e9b0d560ebd8876e408 Mon Sep 17 00:00:00 2001 From: Aakash Pydi Date: Thu, 11 Jun 2020 19:40:58 -0500 Subject: [PATCH 13/18] feature: Support for multi variant endpoint invocation with target variant param (#1577) Co-authored-by: Chuyang --- setup.py | 2 +- src/sagemaker/local/local_session.py | 4 + src/sagemaker/predictor.py | 13 +- tests/integ/test_multi_variant_endpoint.py | 318 +++++++++++++++++++++ tests/unit/test_predictor.py | 26 ++ 5 files changed, 359 insertions(+), 4 deletions(-) create mode 100644 tests/integ/test_multi_variant_endpoint.py diff --git a/setup.py b/setup.py index 9b4c16cf27..8bb6525e97 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def read_version(): # Declare minimal set for installation required_packages = [ - "boto3>=1.13.6", + "boto3>=1.13.24", "numpy>=1.9.0", "protobuf>=3.1", "scipy>=0.19.0", diff --git a/src/sagemaker/local/local_session.py b/src/sagemaker/local/local_session.py index 1e80f6e1b4..6c23cea45a 100644 --- a/src/sagemaker/local/local_session.py +++ b/src/sagemaker/local/local_session.py @@ -343,6 +343,7 @@ def invoke_endpoint( Accept=None, CustomAttributes=None, TargetModel=None, + TargetVariant=None, ): """ @@ -370,6 +371,9 @@ def invoke_endpoint( if TargetModel is not None: headers["X-Amzn-SageMaker-Target-Model"] = TargetModel + if TargetVariant is not None: + headers["X-Amzn-SageMaker-Target-Variant"] = TargetVariant + r = self.http.request("POST", url, body=Body, preload_content=False, headers=headers) return {"Body": r, "ContentType": Accept} diff --git a/src/sagemaker/predictor.py b/src/sagemaker/predictor.py index 80da8a551c..0f1efb4e18 100644 --- a/src/sagemaker/predictor.py +++ b/src/sagemaker/predictor.py @@ -83,7 +83,7 @@ def __init__( self._endpoint_config_name = self._get_endpoint_config_name() self._model_names = self._get_model_names() - def predict(self, data, initial_args=None, target_model=None): + def predict(self, data, initial_args=None, target_model=None, target_variant=None): """Return the inference from the specified endpoint. Args: @@ -98,6 +98,9 @@ def predict(self, data, initial_args=None, target_model=None): target_model (str): S3 model artifact path to run an inference request on, in case of a multi model endpoint. Does not apply to endpoints hosting single model (Default: None) + target_variant (str): The name of the production variant to run an inference + request on (Default: None). Note that the ProductionVariant identifies the model + you want to host and the resources you want to deploy for hosting it. Returns: object: Inference for the given input. If a deserializer was specified when creating @@ -106,7 +109,7 @@ def predict(self, data, initial_args=None, target_model=None): as is. """ - request_args = self._create_request_args(data, initial_args, target_model) + request_args = self._create_request_args(data, initial_args, target_model, target_variant) response = self.sagemaker_session.sagemaker_runtime_client.invoke_endpoint(**request_args) return self._handle_response(response) @@ -123,12 +126,13 @@ def _handle_response(self, response): response_body.close() return data - def _create_request_args(self, data, initial_args=None, target_model=None): + def _create_request_args(self, data, initial_args=None, target_model=None, target_variant=None): """ Args: data: initial_args: target_model: + target_variant: """ args = dict(initial_args) if initial_args else {} @@ -144,6 +148,9 @@ def _create_request_args(self, data, initial_args=None, target_model=None): if target_model: args["TargetModel"] = target_model + if target_variant: + args["TargetVariant"] = target_variant + if self.serializer is not None: data = self.serializer(data) diff --git a/tests/integ/test_multi_variant_endpoint.py b/tests/integ/test_multi_variant_endpoint.py new file mode 100644 index 0000000000..eddcc11768 --- /dev/null +++ b/tests/integ/test_multi_variant_endpoint.py @@ -0,0 +1,318 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from __future__ import absolute_import + +import json +import os +import math +import pytest +import scipy.stats as st + +from sagemaker.s3 import S3Uploader +from sagemaker.session import production_variant +from sagemaker.sparkml import SparkMLModel +from sagemaker.utils import sagemaker_timestamp +from sagemaker.content_types import CONTENT_TYPE_CSV +from sagemaker.utils import unique_name_from_base +from sagemaker.amazon.amazon_estimator import get_image_uri +from sagemaker.predictor import csv_serializer, RealTimePredictor + + +import tests.integ + + +ROLE = "SageMakerRole" +MODEL_NAME = "test-xgboost-model-{}".format(sagemaker_timestamp()) +DEFAULT_REGION = "us-west-2" +DEFAULT_INSTANCE_TYPE = "ml.m5.xlarge" +DEFAULT_INSTANCE_COUNT = 1 +XG_BOOST_MODEL_LOCAL_PATH = os.path.join(tests.integ.DATA_DIR, "xgboost_model", "xgb_model.tar.gz") + +TEST_VARIANT_1 = "Variant1" +TEST_VARIANT_1_WEIGHT = 0.3 + +TEST_VARIANT_2 = "Variant2" +TEST_VARIANT_2_WEIGHT = 0.7 + +VARIANT_TRAFFIC_SAMPLING_COUNT = 100 +DESIRED_CONFIDENCE_FOR_VARIANT_TRAFFIC_DISTRIBUTION = 0.999 + +TEST_CSV_DATA = "42,42,42,42,42,42,42" + +SPARK_ML_MODEL_LOCAL_PATH = os.path.join( + tests.integ.DATA_DIR, "sparkml_model", "mleap_model.tar.gz" +) +SPARK_ML_DEFAULT_VARIANT_NAME = ( + "AllTraffic" +) # default defined in src/sagemaker/session.py def production_variant +SPARK_ML_WRONG_VARIANT_NAME = "WRONG_VARIANT" +SPARK_ML_TEST_DATA = "1.0,C,38.0,71.5,1.0,female" +SPARK_ML_MODEL_SCHEMA = json.dumps( + { + "input": [ + {"name": "Pclass", "type": "float"}, + {"name": "Embarked", "type": "string"}, + {"name": "Age", "type": "float"}, + {"name": "Fare", "type": "float"}, + {"name": "SibSp", "type": "float"}, + {"name": "Sex", "type": "string"}, + ], + "output": {"name": "features", "struct": "vector", "type": "double"}, + } +) + + +@pytest.fixture(scope="module") +def multi_variant_endpoint(sagemaker_session): + """ + Sets up the multi variant endpoint before the integration tests run. + Cleans up the multi variant endpoint after the integration tests run. + """ + multi_variant_endpoint.endpoint_name = unique_name_from_base( + "integ-test-multi-variant-endpoint" + ) + with tests.integ.timeout.timeout_and_delete_endpoint_by_name( + endpoint_name=multi_variant_endpoint.endpoint_name, + sagemaker_session=sagemaker_session, + hours=2, + ): + + # Creating a model + bucket = sagemaker_session.default_bucket() + prefix = "sagemaker/DEMO-VariantTargeting" + model_url = S3Uploader.upload( + local_path=XG_BOOST_MODEL_LOCAL_PATH, + desired_s3_uri="s3://" + bucket + "/" + prefix, + session=sagemaker_session, + ) + + image_uri = get_image_uri(sagemaker_session.boto_session.region_name, "xgboost", "0.90-1") + + multi_variant_endpoint_model = sagemaker_session.create_model( + name=MODEL_NAME, + role=ROLE, + container_defs={"Image": image_uri, "ModelDataUrl": model_url}, + ) + + # Creating a multi variant endpoint + variant1 = production_variant( + model_name=MODEL_NAME, + instance_type=DEFAULT_INSTANCE_TYPE, + initial_instance_count=DEFAULT_INSTANCE_COUNT, + variant_name=TEST_VARIANT_1, + initial_weight=TEST_VARIANT_1_WEIGHT, + ) + variant2 = production_variant( + model_name=MODEL_NAME, + instance_type=DEFAULT_INSTANCE_TYPE, + initial_instance_count=DEFAULT_INSTANCE_COUNT, + variant_name=TEST_VARIANT_2, + initial_weight=TEST_VARIANT_2_WEIGHT, + ) + sagemaker_session.endpoint_from_production_variants( + name=multi_variant_endpoint.endpoint_name, production_variants=[variant1, variant2] + ) + + # Yield to run the integration tests + yield multi_variant_endpoint + + # Cleanup resources + sagemaker_session.delete_model(multi_variant_endpoint_model) + sagemaker_session.sagemaker_client.delete_endpoint_config( + EndpointConfigName=multi_variant_endpoint.endpoint_name + ) + + # Validate resource cleanup + with pytest.raises(Exception) as exception: + sagemaker_session.sagemaker_client.describe_model( + ModelName=multi_variant_endpoint_model.name + ) + assert "Could not find model" in str(exception.value) + sagemaker_session.sagemaker_client.describe_endpoint_config( + name=multi_variant_endpoint.endpoint_name + ) + assert "Could not find endpoint" in str(exception.value) + + +def test_target_variant_invocation(sagemaker_session, multi_variant_endpoint): + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=multi_variant_endpoint.endpoint_name, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_1, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_1 + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=multi_variant_endpoint.endpoint_name, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_2, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_2 + + +def test_predict_invocation_with_target_variant(sagemaker_session, multi_variant_endpoint): + predictor = RealTimePredictor( + endpoint=multi_variant_endpoint.endpoint_name, + sagemaker_session=sagemaker_session, + serializer=csv_serializer, + content_type=CONTENT_TYPE_CSV, + accept=CONTENT_TYPE_CSV, + ) + + # Validate that no exception is raised when the target_variant is specified. + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2) + + +def test_variant_traffic_distribution(sagemaker_session, multi_variant_endpoint): + variant_1_invocation_count = 0 + variant_2_invocation_count = 0 + + for i in range(0, VARIANT_TRAFFIC_SAMPLING_COUNT): + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=multi_variant_endpoint.endpoint_name, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + ) + if response["InvokedProductionVariant"] == TEST_VARIANT_1: + variant_1_invocation_count += 1 + elif response["InvokedProductionVariant"] == TEST_VARIANT_2: + variant_2_invocation_count += 1 + + assert variant_1_invocation_count + variant_2_invocation_count == VARIANT_TRAFFIC_SAMPLING_COUNT + + variant_1_invocation_percentage = float(variant_1_invocation_count) / float( + VARIANT_TRAFFIC_SAMPLING_COUNT + ) + variant_1_margin_of_error = _compute_and_retrieve_margin_of_error(TEST_VARIANT_1_WEIGHT) + assert variant_1_invocation_percentage < TEST_VARIANT_1_WEIGHT + variant_1_margin_of_error + assert variant_1_invocation_percentage > TEST_VARIANT_1_WEIGHT - variant_1_margin_of_error + + variant_2_invocation_percentage = float(variant_2_invocation_count) / float( + VARIANT_TRAFFIC_SAMPLING_COUNT + ) + variant_2_margin_of_error = _compute_and_retrieve_margin_of_error(TEST_VARIANT_2_WEIGHT) + assert variant_2_invocation_percentage < TEST_VARIANT_2_WEIGHT + variant_2_margin_of_error + assert variant_2_invocation_percentage > TEST_VARIANT_2_WEIGHT - variant_2_margin_of_error + + +def test_spark_ml_predict_invocation_with_target_variant(sagemaker_session): + + spark_ml_model_endpoint_name = unique_name_from_base("integ-test-target-variant-sparkml") + + model_data = sagemaker_session.upload_data( + path=SPARK_ML_MODEL_LOCAL_PATH, key_prefix="integ-test-data/sparkml/model" + ) + + with tests.integ.timeout.timeout_and_delete_endpoint_by_name( + spark_ml_model_endpoint_name, sagemaker_session + ): + spark_ml_model = SparkMLModel( + model_data=model_data, + role=ROLE, + sagemaker_session=sagemaker_session, + env={"SAGEMAKER_SPARKML_SCHEMA": SPARK_ML_MODEL_SCHEMA}, + ) + + predictor = spark_ml_model.deploy( + DEFAULT_INSTANCE_COUNT, + DEFAULT_INSTANCE_TYPE, + endpoint_name=spark_ml_model_endpoint_name, + ) + + # Validate that no exception is raised when the target_variant is specified. + predictor.predict(SPARK_ML_TEST_DATA, target_variant=SPARK_ML_DEFAULT_VARIANT_NAME) + + with pytest.raises(Exception) as exception_info: + predictor.predict(SPARK_ML_TEST_DATA, target_variant=SPARK_ML_WRONG_VARIANT_NAME) + + assert "ValidationError" in str(exception_info.value) + assert SPARK_ML_WRONG_VARIANT_NAME in str(exception_info.value) + + # cleanup resources + spark_ml_model.delete_model() + sagemaker_session.sagemaker_client.delete_endpoint_config( + EndpointConfigName=spark_ml_model_endpoint_name + ) + + # Validate resource cleanup + with pytest.raises(Exception) as exception: + sagemaker_session.sagemaker_client.describe_model(ModelName=spark_ml_model.name) + assert "Could not find model" in str(exception.value) + sagemaker_session.sagemaker_client.describe_endpoint_config( + name=spark_ml_model_endpoint_name + ) + assert "Could not find endpoint" in str(exception.value) + + +@pytest.mark.local_mode +def test_target_variant_invocation_local_mode(sagemaker_session, multi_variant_endpoint): + + if sagemaker_session._region_name is None: + sagemaker_session._region_name = DEFAULT_REGION + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=multi_variant_endpoint.endpoint_name, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_1, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_1 + + response = sagemaker_session.sagemaker_runtime_client.invoke_endpoint( + EndpointName=multi_variant_endpoint.endpoint_name, + Body=TEST_CSV_DATA, + ContentType=CONTENT_TYPE_CSV, + Accept=CONTENT_TYPE_CSV, + TargetVariant=TEST_VARIANT_2, + ) + assert response["InvokedProductionVariant"] == TEST_VARIANT_2 + + +@pytest.mark.local_mode +def test_predict_invocation_with_target_variant_local_mode( + sagemaker_session, multi_variant_endpoint +): + + if sagemaker_session._region_name is None: + sagemaker_session._region_name = DEFAULT_REGION + + predictor = RealTimePredictor( + endpoint=multi_variant_endpoint.endpoint_name, + sagemaker_session=sagemaker_session, + serializer=csv_serializer, + content_type=CONTENT_TYPE_CSV, + accept=CONTENT_TYPE_CSV, + ) + + # Validate that no exception is raised when the target_variant is specified. + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) + predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2) + + +def _compute_and_retrieve_margin_of_error(variant_weight): + """ + Computes the margin of error using the Wald method for computing the confidence + intervals of a binomial distribution. + """ + z_value = st.norm.ppf(DESIRED_CONFIDENCE_FOR_VARIANT_TRAFFIC_DISTRIBUTION) + margin_of_error = (variant_weight * (1 - variant_weight)) / VARIANT_TRAFFIC_SAMPLING_COUNT + margin_of_error = z_value * math.sqrt(margin_of_error) + return margin_of_error diff --git a/tests/unit/test_predictor.py b/tests/unit/test_predictor.py index 9c7feb0a83..b74f36e2b3 100644 --- a/tests/unit/test_predictor.py +++ b/tests/unit/test_predictor.py @@ -346,6 +346,7 @@ def test_numpy_deser_from_npy_object_array(): CSV_CONTENT_TYPE = "text/csv" RETURN_VALUE = 0 CSV_RETURN_VALUE = "1,2,3\r\n" +PRODUCTION_VARIANT_1 = "PRODUCTION_VARIANT_1" ENDPOINT_DESC = {"EndpointConfigName": ENDPOINT} @@ -407,6 +408,31 @@ def test_predict_call_with_headers(): assert result == RETURN_VALUE +def test_predict_call_with_target_variant(): + sagemaker_session = empty_sagemaker_session() + predictor = RealTimePredictor( + ENDPOINT, sagemaker_session, content_type=DEFAULT_CONTENT_TYPE, accept=DEFAULT_CONTENT_TYPE + ) + + data = "untouched" + result = predictor.predict(data, target_variant=PRODUCTION_VARIANT_1) + + assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called + + expected_request_args = { + "Accept": DEFAULT_CONTENT_TYPE, + "Body": data, + "ContentType": DEFAULT_CONTENT_TYPE, + "EndpointName": ENDPOINT, + "TargetVariant": PRODUCTION_VARIANT_1, + } + + call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args + assert kwargs == expected_request_args + + assert result == RETURN_VALUE + + def test_multi_model_predict_call_with_headers(): sagemaker_session = empty_sagemaker_session() predictor = RealTimePredictor( From 974b734a538efa35b7a00ad079d2aa85d8676551 Mon Sep 17 00:00:00 2001 From: Piali Das <32551277+pdasamzn@users.noreply.github.com> Date: Fri, 12 Jun 2020 14:46:44 -0400 Subject: [PATCH 14/18] feature: Allow selecting inference response content for automl generated models (#1566) * Added attach() method to AutoML to allow attaching an AutoML object to an existing AutoMLJob * Added create_model() method to AutoML that returns a PipelineModel that can be used by the * Added a util function validate_and_update_inference_response() to update the inference container list based on the requested inference response keys * Updated and added unit tests --- src/sagemaker/automl/automl.py | 362 ++++++++++++++------ tests/data/automl/data/iris_transform.csv | 15 + tests/integ/test_auto_ml.py | 71 ++++ tests/unit/sagemaker/automl/test_auto_ml.py | 188 ++++++++-- 4 files changed, 506 insertions(+), 130 deletions(-) create mode 100644 tests/data/automl/data/iris_transform.csv diff --git a/src/sagemaker/automl/automl.py b/src/sagemaker/automl/automl.py index 794e03aee1..c204356e5b 100644 --- a/src/sagemaker/automl/automl.py +++ b/src/sagemaker/automl/automl.py @@ -100,6 +100,67 @@ def fit(self, inputs=None, wait=True, logs=True, job_name=None): if wait: self.latest_auto_ml_job.wait(logs=logs) + @classmethod + def attach(cls, auto_ml_job_name, sagemaker_session=None): + """Attach to an existing AutoML job. + + Creates and returns a AutoML bound to an existing automl job. + + Args: + auto_ml_job_name (str): AutoML job name + sagemaker_session (sagemaker.session.Session): A SageMaker Session + object, used for SageMaker interactions (default: None). If not + specified, the one originally associated with the ``AutoML`` instance is used. + + Returns: + sagemaker.automl.AutoML: A ``AutoML`` instance with the attached automl job. + + """ + sagemaker_session = sagemaker_session or Session() + + auto_ml_job_desc = sagemaker_session.describe_auto_ml_job(auto_ml_job_name) + automl_job_tags = sagemaker_session.sagemaker_client.list_tags( + ResourceArn=auto_ml_job_desc["AutoMLJobArn"] + )["Tags"] + + amlj = AutoML( + role=auto_ml_job_desc["RoleArn"], + target_attribute_name=auto_ml_job_desc["InputDataConfig"][0]["TargetAttributeName"], + output_kms_key=auto_ml_job_desc["OutputDataConfig"].get("KmsKeyId"), + output_path=auto_ml_job_desc["OutputDataConfig"]["S3OutputPath"], + base_job_name=auto_ml_job_name, + compression_type=auto_ml_job_desc["InputDataConfig"][0].get("CompressionType"), + sagemaker_session=sagemaker_session, + volume_kms_key=auto_ml_job_desc.get("AutoMLJobConfig", {}) + .get("SecurityConfig", {}) + .get("VolumeKmsKeyId"), + encrypt_inter_container_traffic=auto_ml_job_desc.get("AutoMLJobConfig", {}) + .get("SecurityConfig", {}) + .get("EnableInterContainerTrafficEncryption", False), + vpc_config=auto_ml_job_desc.get("AutoMLJobConfig", {}) + .get("SecurityConfig", {}) + .get("VpcConfig"), + problem_type=auto_ml_job_desc.get("ProblemType"), + max_candidates=auto_ml_job_desc.get("AutoMLJobConfig", {}) + .get("CompletionCriteria", {}) + .get("MaxCandidates"), + max_runtime_per_training_job_in_seconds=auto_ml_job_desc.get("AutoMLJobConfig", {}) + .get("CompletionCriteria", {}) + .get("MaxRuntimePerTrainingJobInSeconds"), + total_job_runtime_in_seconds=auto_ml_job_desc.get("AutoMLJobConfig", {}) + .get("CompletionCriteria", {}) + .get("MaxAutoMLJobRuntimeInSeconds"), + job_objective=auto_ml_job_desc.get("AutoMLJobObjective", {}).get("MetricName"), + generate_candidate_definitions_only=auto_ml_job_desc.get( + "GenerateCandidateDefinitionsOnly", False + ), + tags=automl_job_tags, + ) + amlj.current_job_name = auto_ml_job_name + amlj.latest_auto_ml_job = auto_ml_job_name # pylint: disable=W0201 + amlj._auto_ml_job_desc = auto_ml_job_desc + return amlj + def describe_auto_ml_job(self, job_name=None): """Returns the job description of an AutoML job for the given job name. @@ -187,49 +248,29 @@ def list_candidates( return self.sagemaker_session.list_candidates(**list_candidates_args)["Candidates"] - def deploy( + def create_model( self, - initial_instance_count, - instance_type, - candidate=None, + name, sagemaker_session=None, - name=None, - endpoint_name=None, - tags=None, - wait=True, - update_endpoint=False, + candidate=None, vpc_config=None, enable_network_isolation=False, model_kms_key=None, predictor_cls=None, + inference_response_keys=None, ): - """Deploy a candidate to a SageMaker Inference Pipeline and return a Predictor + """Creates a model from a given candidate or the best candidate + from the automl job Args: - initial_instance_count (int): The initial number of instances to run - in the ``Endpoint`` created from this ``Model``. - instance_type (str): The EC2 instance type to deploy this Model to. - For example, 'ml.p2.xlarge'. + name (str): The pipeline model name. + sagemaker_session (sagemaker.session.Session): A SageMaker Session + object, used for SageMaker interactions (default: None). If not + specified, the one originally associated with the ``AutoML`` instance is used.: candidate (CandidateEstimator or dict): a CandidateEstimator used for deploying to a SageMaker Inference Pipeline. If None, the best candidate will be used. If the candidate input is a dict, a CandidateEstimator will be created from it. - sagemaker_session (sagemaker.session.Session): A SageMaker Session - object, used for SageMaker interactions (default: None). If not - specified, the one originally associated with the ``AutoML`` instance is used. - name (str): The pipeline model name. If None, a default model name will - be selected on each ``deploy``. - endpoint_name (str): The name of the endpoint to create (default: - None). If not specified, a unique endpoint name will be created. - tags (List[dict[str, str]]): The list of tags to attach to this - specific endpoint. - wait (bool): Whether the call should wait until the deployment of - model completes (default: True). - update_endpoint (bool): Flag to update the model in an existing - Amazon SageMaker endpoint. If True, this will deploy a new - EndpointConfig to an already existing endpoint and delete - resources corresponding to the previous EndpointConfig. If - False, a new endpoint will be created. Default: False vpc_config (dict): Specifies a VPC that your training jobs and hosted models have access to. Contents include "SecurityGroupIds" and "Subnets". enable_network_isolation (bool): Isolates the training container. No inbound or @@ -241,11 +282,12 @@ def deploy( function to call to create a predictor (default: None). If specified, ``deploy()`` returns the result of invoking this function on the created endpoint name. + inference_response_keys (list): List of keys for response content. The order of the + keys will dictate the content order in the response. Returns: - callable[string, sagemaker.session.Session] or ``None``: - If ``predictor_cls`` is specified, the invocation of ``self.predictor_cls`` on - the created endpoint name. Otherwise, ``None``. + PipelineModel object + """ sagemaker_session = sagemaker_session or self.sagemaker_session @@ -256,50 +298,46 @@ def deploy( candidate = CandidateEstimator(candidate, sagemaker_session=sagemaker_session) inference_containers = candidate.containers - endpoint_name = endpoint_name or self.current_job_name - - return self._deploy_inference_pipeline( - inference_containers, - initial_instance_count=initial_instance_count, - instance_type=instance_type, - name=name, - sagemaker_session=sagemaker_session, - endpoint_name=endpoint_name, - tags=tags, - wait=wait, - update_endpoint=update_endpoint, - vpc_config=vpc_config, - enable_network_isolation=enable_network_isolation, - model_kms_key=model_kms_key, - predictor_cls=predictor_cls, - ) - def _check_problem_type_and_job_objective(self, problem_type, job_objective): - """Validate if problem_type and job_objective are both None or are both provided. + self.validate_and_update_inference_response(inference_containers, inference_response_keys) - Args: - problem_type (str): The type of problem of this AutoMLJob. Valid values are - "Regression", "BinaryClassification", "MultiClassClassification". - job_objective (dict): AutoMLJob objective, contains "AutoMLJobObjectiveType" (optional), - "MetricName" and "Value". + # construct Model objects + models = [] - Raises (ValueError): raises ValueError if one of problem_type and job_objective is provided - while the other is None. + for container in inference_containers: + image = container["Image"] + model_data = container["ModelDataUrl"] + env = container["Environment"] - """ - if not (problem_type and job_objective) and (problem_type or job_objective): - raise ValueError( - "One of problem type and objective metric provided. " - "Either both of them should be provided or none of them should be provided." + model = Model( + image=image, + model_data=model_data, + role=self.role, + env=env, + vpc_config=vpc_config, + sagemaker_session=sagemaker_session or self.sagemaker_session, + enable_network_isolation=enable_network_isolation, + model_kms_key=model_kms_key, ) + models.append(model) + + pipeline = PipelineModel( + models=models, + role=self.role, + predictor_cls=predictor_cls, + name=name, + vpc_config=vpc_config, + sagemaker_session=sagemaker_session or self.sagemaker_session, + ) + return pipeline - def _deploy_inference_pipeline( + def deploy( self, - inference_containers, initial_instance_count, instance_type, - name=None, + candidate=None, sagemaker_session=None, + name=None, endpoint_name=None, tags=None, wait=True, @@ -308,21 +346,24 @@ def _deploy_inference_pipeline( enable_network_isolation=False, model_kms_key=None, predictor_cls=None, + inference_response_keys=None, ): - """Deploy a SageMaker Inference Pipeline. + """Deploy a candidate to a SageMaker Inference Pipeline and return a Predictor Args: - inference_containers (list): a list of inference container definitions initial_instance_count (int): The initial number of instances to run in the ``Endpoint`` created from this ``Model``. instance_type (str): The EC2 instance type to deploy this Model to. For example, 'ml.p2.xlarge'. - name (str): The pipeline model name. If None, a default model name will - be selected on each ``deploy``. + candidate (CandidateEstimator or dict): a CandidateEstimator used for deploying + to a SageMaker Inference Pipeline. If None, the best candidate will + be used. If the candidate input is a dict, a CandidateEstimator will be + created from it. sagemaker_session (sagemaker.session.Session): A SageMaker Session object, used for SageMaker interactions (default: None). If not - specified, one is created using the default AWS configuration - chain. + specified, the one originally associated with the ``AutoML`` instance is used. + name (str): The pipeline model name. If None, a default model name will + be selected on each ``deploy``. endpoint_name (str): The name of the endpoint to create (default: None). If not specified, a unique endpoint name will be created. tags (List[dict[str, str]]): The list of tags to attach to this @@ -334,44 +375,38 @@ def _deploy_inference_pipeline( EndpointConfig to an already existing endpoint and delete resources corresponding to the previous EndpointConfig. If False, a new endpoint will be created. Default: False - vpc_config (dict): information about vpc configuration, optionally - contains "SecurityGroupIds", "Subnets" + vpc_config (dict): Specifies a VPC that your training jobs and hosted models have + access to. Contents include "SecurityGroupIds" and "Subnets". + enable_network_isolation (bool): Isolates the training container. No inbound or + outbound network calls can be made, except for calls between peers within a + training cluster for distributed training. Default: False model_kms_key (str): KMS key ARN used to encrypt the repacked model archive file if the model is repacked predictor_cls (callable[string, sagemaker.session.Session]): A function to call to create a predictor (default: None). If specified, ``deploy()`` returns the result of invoking this function on the created endpoint name. - """ - # construct Model objects - models = [] - for container in inference_containers: - image = container["Image"] - model_data = container["ModelDataUrl"] - env = container["Environment"] + inference_response_keys (list): List of keys for response content. The order of the + keys will dictate the content order in the response. - model = Model( - image=image, - model_data=model_data, - role=self.role, - env=env, - vpc_config=vpc_config, - sagemaker_session=sagemaker_session or self.sagemaker_session, - enable_network_isolation=enable_network_isolation, - model_kms_key=model_kms_key, - ) - models.append(model) - - pipeline = PipelineModel( - models=models, - role=self.role, - predictor_cls=predictor_cls, + Returns: + callable[string, sagemaker.session.Session] or ``None``: + If ``predictor_cls`` is specified, the invocation of ``self.predictor_cls`` on + the created endpoint name. Otherwise, ``None``. + """ + sagemaker_session = sagemaker_session or self.sagemaker_session + model = self.create_model( name=name, + sagemaker_session=sagemaker_session, + candidate=candidate, + inference_response_keys=inference_response_keys, vpc_config=vpc_config, - sagemaker_session=sagemaker_session or self.sagemaker_session, + enable_network_isolation=enable_network_isolation, + model_kms_key=model_kms_key, + predictor_cls=predictor_cls, ) - return pipeline.deploy( + return model.deploy( initial_instance_count=initial_instance_count, instance_type=instance_type, endpoint_name=endpoint_name, @@ -380,6 +415,25 @@ def _deploy_inference_pipeline( update_endpoint=update_endpoint, ) + def _check_problem_type_and_job_objective(self, problem_type, job_objective): + """Validate if problem_type and job_objective are both None or are both provided. + + Args: + problem_type (str): The type of problem of this AutoMLJob. Valid values are + "Regression", "BinaryClassification", "MultiClassClassification". + job_objective (dict): AutoMLJob objective, contains "AutoMLJobObjectiveType" (optional), + "MetricName" and "Value". + + Raises (ValueError): raises ValueError if one of problem_type and job_objective is provided + while the other is None. + + """ + if not (problem_type and job_objective) and (problem_type or job_objective): + raise ValueError( + "One of problem type and objective metric provided. " + "Either both of them should be provided or none of them should be provided." + ) + def _prepare_for_auto_ml_job(self, job_name=None): """Set any values in the AutoMLJob that need to be set before creating request. @@ -400,6 +454,114 @@ def _prepare_for_auto_ml_job(self, job_name=None): if self.output_path is None: self.output_path = "s3://{}/".format(self.sagemaker_session.default_bucket()) + @classmethod + def _get_supported_inference_keys(cls, container, default=None): + """Returns the inference keys supported by the container. + + Args: + container (dict): Dictionary representing container + default (object): The value to be returned if the container definition + has no marker environment variable + + Returns: + List of keys the container support or default + + Raises: + KeyError if the default is None and the container definition has + no marker environment variable SAGEMAKER_INFERENCE_SUPPORTED. + """ + try: + return [ + x.strip() + for x in container["Environment"]["SAGEMAKER_INFERENCE_SUPPORTED"].split(",") + ] + except KeyError: + if default is None: + raise + return default + + @classmethod + def _check_inference_keys(cls, inference_response_keys, containers): + """Given an inference container list, checks if the pipeline supports the + requested inference keys + + Args: + inference_response_keys (list): List of keys for inference response content + containers (list): list of inference container + + Raises: + ValueError, if one or more keys in inference_response_keys are not supported + the inference pipeline. + + """ + if not inference_response_keys: + return + try: + supported_inference_keys = cls._get_supported_inference_keys(container=containers[-1]) + except KeyError: + raise ValueError( + "The inference model does not support selection of inference content beyond " + "it's default content. Please retry without setting " + "inference_response_keys key word argument." + ) + bad_keys = [] + for key in inference_response_keys: + if key not in supported_inference_keys: + bad_keys.append(key) + + if bad_keys: + raise ValueError( + "Requested inference output keys [{bad_keys_str}] are unsupported. " + "The supported inference keys are [{allowed_keys_str}]".format( + bad_keys_str=", ".join(bad_keys), + allowed_keys_str=", ".join(supported_inference_keys), + ) + ) + + @classmethod + def validate_and_update_inference_response(cls, inference_containers, inference_response_keys): + """Validates the requested inference keys and updates inference containers to emit the + requested content in the inference response. + + Args: + inference_containers (list): list of inference containers + inference_response_keys (list): list of inference response keys + + Raises: + ValueError: if one or more of inference_response_keys are unsupported by the model + + """ + if not inference_response_keys: + return + + cls._check_inference_keys(inference_response_keys, inference_containers) + + previous_container_output = None + + for container in inference_containers: + supported_inference_keys_container = cls._get_supported_inference_keys( + container, default=[] + ) + if not supported_inference_keys_container: + previous_container_output = None + continue + current_container_output = None + for key in inference_response_keys: + if key in supported_inference_keys_container: + current_container_output = ( + current_container_output + "," + key if current_container_output else key + ) + + if previous_container_output: + container["Environment"].update( + {"SAGEMAKER_INFERENCE_INPUT": previous_container_output} + ) + if current_container_output: + container["Environment"].update( + {"SAGEMAKER_INFERENCE_OUTPUT": current_container_output} + ) + previous_container_output = current_container_output + class AutoMLInput(object): """Accepts parameters that specify an S3 input for an auto ml job and provides diff --git a/tests/data/automl/data/iris_transform.csv b/tests/data/automl/data/iris_transform.csv new file mode 100644 index 0000000000..114f823d28 --- /dev/null +++ b/tests/data/automl/data/iris_transform.csv @@ -0,0 +1,15 @@ +6.4,2.8,5.6,2.2 +5.0,2.3,3.3,1.0 +4.9,2.5,4.5,1.7 +4.9,3.1,1.5,0.1 +5.7,3.8,1.7,0.3 +4.4,3.2,1.3,0.2 +5.4,3.4,1.5,0.4 +6.9,3.1,5.1,2.3 +6.7,3.1,4.4,1.4 +5.1,3.7,1.5,0.4 +5.2,2.7,3.9,1.4 +6.9,3.1,4.9,1.5 +5.8,4.0,1.2,0.2 +5.4,3.9,1.7,0.4 +7.7,3.8,6.7,2.2 \ No newline at end of file diff --git a/tests/integ/test_auto_ml.py b/tests/integ/test_auto_ml.py index 26e90ecf74..fc547256c6 100644 --- a/tests/integ/test_auto_ml.py +++ b/tests/integ/test_auto_ml.py @@ -32,11 +32,14 @@ DATA_DIR = os.path.join(DATA_DIR, "automl", "data") TRAINING_DATA = os.path.join(DATA_DIR, "iris_training.csv") TEST_DATA = os.path.join(DATA_DIR, "iris_test.csv") +TRANSFORM_DATA = os.path.join(DATA_DIR, "iris_transform.csv") PROBLEM_TYPE = "MultiClassClassification" BASE_JOB_NAME = "auto-ml" # use a succeeded AutoML job to test describe and list candidates method, otherwise tests will run too long AUTO_ML_JOB_NAME = "python-sdk-integ-test-base-job" +DEFAULT_MODEL_NAME = "python-sdk-automl" + EXPECTED_DEFAULT_JOB_CONFIG = { "CompletionCriteria": {"MaxCandidates": 3}, @@ -180,6 +183,42 @@ def test_auto_ml_describe_auto_ml_job(sagemaker_session): assert desc["OutputDataConfig"] == expected_default_output_config +@pytest.mark.skipif( + tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS, + reason="AutoML is not supported in the region yet.", +) +def test_auto_ml_attach(sagemaker_session): + expected_default_input_config = [ + { + "DataSource": { + "S3DataSource": { + "S3DataType": "S3Prefix", + "S3Uri": "s3://{}/{}/input/iris_training.csv".format( + sagemaker_session.default_bucket(), PREFIX + ), + } + }, + "TargetAttributeName": TARGET_ATTRIBUTE_NAME, + } + ] + expected_default_output_config = { + "S3OutputPath": "s3://{}/".format(sagemaker_session.default_bucket()) + } + + auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session) + + attached_automl_job = AutoML.attach( + auto_ml_job_name=AUTO_ML_JOB_NAME, sagemaker_session=sagemaker_session + ) + attached_desc = attached_automl_job.describe_auto_ml_job() + assert attached_desc["AutoMLJobName"] == AUTO_ML_JOB_NAME + assert attached_desc["AutoMLJobStatus"] == "Completed" + assert isinstance(attached_desc["BestCandidate"], dict) + assert attached_desc["InputDataConfig"] == expected_default_input_config + assert attached_desc["AutoMLJobConfig"] == EXPECTED_DEFAULT_JOB_CONFIG + assert attached_desc["OutputDataConfig"] == expected_default_output_config + + @pytest.mark.skipif( tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS, reason="AutoML is not supported in the region yet.", @@ -240,6 +279,38 @@ def test_deploy_best_candidate(sagemaker_session, cpu_instance_type): sagemaker_session.sagemaker_client.delete_endpoint(EndpointName=endpoint_name) +@pytest.mark.skipif( + tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS, + reason="AutoML is not supported in the region yet.", +) +def test_create_model_best_candidate(sagemaker_session, cpu_instance_type): + auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session) + + auto_ml = AutoML.attach(auto_ml_job_name=AUTO_ML_JOB_NAME, sagemaker_session=sagemaker_session) + best_candidate = auto_ml.best_candidate() + + with timeout(minutes=5): + pipeline_model = auto_ml.create_model( + name=DEFAULT_MODEL_NAME, + candidate=best_candidate, + sagemaker_session=sagemaker_session, + vpc_config=None, + enable_network_isolation=False, + model_kms_key=None, + predictor_cls=None, + ) + inputs = sagemaker_session.upload_data( + path=TRANSFORM_DATA, key_prefix=PREFIX + "/transform_input" + ) + pipeline_model.transformer( + instance_count=1, + instance_type=cpu_instance_type, + assemble_with="Line", + output_path="s3://{}/{}".format(sagemaker_session.default_bucket(), "transform_test"), + accept="text/csv", + ).transform(data=inputs, content_type="text/csv", split_type="Line", join_source="Input") + + @pytest.mark.skipif( tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS, reason="AutoML is not supported in the region yet.", diff --git a/tests/unit/sagemaker/automl/test_auto_ml.py b/tests/unit/sagemaker/automl/test_auto_ml.py index 8ef0cd31da..70adc840bc 100644 --- a/tests/unit/sagemaker/automl/test_auto_ml.py +++ b/tests/unit/sagemaker/automl/test_auto_ml.py @@ -12,9 +12,11 @@ # language governing permissions and limitations under the License. from __future__ import absolute_import +import copy + import pytest from mock import Mock, patch -from sagemaker import AutoML, AutoMLJob, AutoMLInput, CandidateEstimator +from sagemaker import AutoML, AutoMLJob, AutoMLInput, CandidateEstimator, PipelineModel from sagemaker.predictor import RealTimePredictor MODEL_DATA = "s3://bucket/model.tar.gz" @@ -37,12 +39,14 @@ JOB_NAME = "default-job-name" JOB_NAME_2 = "banana-auto-ml-job" +JOB_NAME_3 = "descriptive-auto-ml-job" VOLUME_KMS_KEY = "volume-kms-key-id-string" OUTPUT_KMS_KEY = "output-kms-key-id-string" OUTPUT_PATH = "s3://my_other_bucket/" BASE_JOB_NAME = "banana" PROBLEM_TYPE = "BinaryClassification" BLACKLISTED_ALGORITHM = ["xgboost"] +LIST_TAGS_RESULT = {"Tags": [{"Key": "key1", "Value": "value1"}]} MAX_CANDIDATES = 10 MAX_RUNTIME_PER_TRAINING_JOB = 3600 TOTAL_JOB_RUNTIME = 36000 @@ -57,6 +61,33 @@ BEST_CANDIDATE_2 = {"best-candidate": "best-trial-2"} AUTO_ML_DESC = {"AutoMLJobName": JOB_NAME, "BestCandidate": BEST_CANDIDATE} AUTO_ML_DESC_2 = {"AutoMLJobName": JOB_NAME_2, "BestCandidate": BEST_CANDIDATE_2} +AUTO_ML_DESC_3 = { + "AutoMLJobArn": "automl_job_arn", + "AutoMLJobConfig": { + "CompletionCriteria": { + "MaxAutoMLJobRuntimeInSeconds": 3000, + "MaxCandidates": 28, + "MaxRuntimePerTrainingJobInSeconds": 100, + }, + "SecurityConfig": {"EnableInterContainerTrafficEncryption": True}, + }, + "AutoMLJobName": "mock_automl_job_name", + "AutoMLJobObjective": {"MetricName": "Auto"}, + "AutoMLJobSecondaryStatus": "Completed", + "AutoMLJobStatus": "Completed", + "GenerateCandidateDefinitionsOnly": False, + "InputDataConfig": [ + { + "DataSource": { + "S3DataSource": {"S3DataType": "S3Prefix", "S3Uri": "s3://input/prefix"} + }, + "TargetAttributeName": "y", + } + ], + "OutputDataConfig": {"KmsKeyId": "string", "S3OutputPath": "s3://output_prefix"}, + "ProblemType": "Auto", + "RoleArn": "mock_role_arn", +} INFERENCE_CONTAINERS = [ { @@ -76,6 +107,33 @@ }, ] +CLASSIFICATION_INFERENCE_CONTAINERS = [ + { + "Environment": {"SAGEMAKER_PROGRAM": "sagemaker_serve"}, + "Image": "account.dkr.ecr.us-west-2.amazonaws.com/sagemaker-auto-ml-data-processing:1.0-cpu-py3", + "ModelDataUrl": "s3://sagemaker-us-west-2-account/sagemaker-auto-ml-gamma/data-processing/output", + }, + { + "Environment": { + "MAX_CONTENT_LENGTH": "20000000", + "SAGEMAKER_INFERENCE_SUPPORTED": "probability,probabilities,predicted_label", + "SAGEMAKER_INFERENCE_OUTPUT": "predicted_label", + }, + "Image": "account.dkr.ecr.us-west-2.amazonaws.com/sagemaker-auto-ml-training:1.0-cpu-py3", + "ModelDataUrl": "s3://sagemaker-us-west-2-account/sagemaker-auto-ml-gamma/training/output", + }, + { + "Environment": { + "INVERSE_LABEL_TRANSFORM": "1", + "SAGEMAKER_INFERENCE_SUPPORTED": "probability,probabilities,predicted_label,labels", + "SAGEMAKER_INFERENCE_OUTPUT": "predicted_label", + "SAGEMAKER_INFERENCE_INPUT": "predicted_label", + }, + "Image": "account.dkr.ecr.us-west-2.amazonaws.com/sagemaker-auto-ml-transform:1.0-cpu-py3", + "ModelDataUrl": "s3://sagemaker-us-west-2-account/sagemaker-auto-ml-gamma/transform/output", + }, +] + CANDIDATE_STEPS = [ { "CandidateStepName": "training-job/sagemaker-auto-ml-gamma/data-processing", @@ -97,6 +155,12 @@ "CandidateSteps": CANDIDATE_STEPS, } +CLASSIFICATION_CANDIDATE_DICT = { + "CandidateName": "candidate_mock", + "InferenceContainers": CLASSIFICATION_INFERENCE_CONTAINERS, + "CandidateSteps": CANDIDATE_STEPS, +} + TRAINING_JOB = { "AlgorithmSpecification": { "AlgorithmName": "string", @@ -143,6 +207,8 @@ def describe_auto_ml_job_mock(job_name=None): return AUTO_ML_DESC elif job_name == JOB_NAME_2: return AUTO_ML_DESC_2 + elif job_name == JOB_NAME_3: + return AUTO_ML_DESC_3 @pytest.fixture() @@ -168,7 +234,7 @@ def sagemaker_session(): name="describe_transform_job", return_value=TRANSFORM_JOB ) sms.list_candidates = Mock(name="list_candidates", return_value={"Candidates": []}) - + sms.sagemaker_client.list_tags = Mock(name="list_tags", return_value=LIST_TAGS_RESULT) return sms @@ -452,29 +518,17 @@ def test_deploy(sagemaker_session, candidate_mock): auto_ml = AutoML( role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session ) + mock_pipeline = Mock(name="pipeline_model") + mock_pipeline.deploy = Mock(name="model_deploy") auto_ml.best_candidate = Mock(name="best_candidate", return_value=CANDIDATE_DICT) - auto_ml._deploy_inference_pipeline = Mock("_deploy_inference_pipeline", return_value=None) + auto_ml.create_model = Mock(name="create_model", return_value=mock_pipeline) auto_ml.deploy( initial_instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, sagemaker_session=sagemaker_session, ) - auto_ml._deploy_inference_pipeline.assert_called_once() - auto_ml._deploy_inference_pipeline.assert_called_with( - candidate_mock.containers, - initial_instance_count=INSTANCE_COUNT, - instance_type=INSTANCE_TYPE, - name=None, - sagemaker_session=sagemaker_session, - endpoint_name=None, - tags=None, - wait=True, - update_endpoint=False, - vpc_config=None, - enable_network_isolation=False, - model_kms_key=None, - predictor_cls=None, - ) + auto_ml.create_model.assert_called_once() + mock_pipeline.deploy.assert_called_once() @patch("sagemaker.automl.automl.CandidateEstimator") @@ -484,7 +538,10 @@ def test_deploy_optional_args(candidate_estimator, sagemaker_session, candidate_ auto_ml = AutoML( role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session ) - auto_ml._deploy_inference_pipeline = Mock("_deploy_inference_pipeline", return_value=None) + mock_pipeline = Mock(name="pipeline_model") + mock_pipeline.deploy = Mock(name="model_deploy") + auto_ml.best_candidate = Mock(name="best_candidate", return_value=CANDIDATE_DICT) + auto_ml.create_model = Mock(name="create_model", return_value=mock_pipeline) auto_ml.deploy( initial_instance_count=INSTANCE_COUNT, @@ -500,25 +557,31 @@ def test_deploy_optional_args(candidate_estimator, sagemaker_session, candidate_ enable_network_isolation=True, model_kms_key=OUTPUT_KMS_KEY, predictor_cls=RealTimePredictor, + inference_response_keys=None, ) - auto_ml._deploy_inference_pipeline.assert_called_once() - auto_ml._deploy_inference_pipeline.assert_called_with( - candidate_mock.containers, - initial_instance_count=INSTANCE_COUNT, - instance_type=INSTANCE_TYPE, + + auto_ml.create_model.assert_called_once() + auto_ml.create_model.assert_called_with( name=JOB_NAME, sagemaker_session=sagemaker_session, - endpoint_name=JOB_NAME, - tags=TAGS, - wait=False, - update_endpoint=True, + candidate=CANDIDATE_DICT, + inference_response_keys=None, vpc_config=VPC_CONFIG, enable_network_isolation=True, model_kms_key=OUTPUT_KMS_KEY, predictor_cls=RealTimePredictor, ) - candidate_estimator.assert_called_with(CANDIDATE_DICT, sagemaker_session=sagemaker_session) + mock_pipeline.deploy.assert_called_once() + + mock_pipeline.deploy.assert_called_with( + initial_instance_count=INSTANCE_COUNT, + instance_type=INSTANCE_TYPE, + endpoint_name=JOB_NAME, + tags=TAGS, + wait=False, + update_endpoint=True, + ) def test_candidate_estimator_get_steps(sagemaker_session): @@ -536,3 +599,68 @@ def test_candidate_estimator_fit(sagemaker_session): candidate_estimator.fit(inputs) sagemaker_session.train.assert_called() sagemaker_session.transform.assert_called() + + +def test_validate_and_update_inference_response(): + cic = copy.copy(CLASSIFICATION_INFERENCE_CONTAINERS) + + AutoML.validate_and_update_inference_response( + inference_containers=cic, + inference_response_keys=["predicted_label", "labels", "probabilities", "probability"], + ) + + assert ( + cic[2]["Environment"]["SAGEMAKER_INFERENCE_OUTPUT"] + == "predicted_label,labels,probabilities,probability" + ) + assert ( + cic[2]["Environment"]["SAGEMAKER_INFERENCE_INPUT"] + == "predicted_label,probabilities,probability" + ) + assert ( + cic[1]["Environment"]["SAGEMAKER_INFERENCE_OUTPUT"] + == "predicted_label,probabilities,probability" + ) + + +def test_validate_and_update_inference_response_wrong_input(): + cic = copy.copy(CLASSIFICATION_INFERENCE_CONTAINERS) + + with pytest.raises( + ValueError, + message="Requested inference output keys [wrong_key, wrong_label] are unsupported. " + "The supported inference keys are [probability, probabilities, predicted_label, labels]", + ): + AutoML.validate_and_update_inference_response( + inference_containers=cic, + inference_response_keys=["wrong_key", "wrong_label", "probabilities", "probability"], + ) + + +def test_create_model(sagemaker_session): + auto_ml = AutoML( + role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session + ) + + pipeline_model = auto_ml.create_model( + name=JOB_NAME, + sagemaker_session=sagemaker_session, + candidate=CLASSIFICATION_CANDIDATE_DICT, + vpc_config=VPC_CONFIG, + enable_network_isolation=True, + model_kms_key=None, + predictor_cls=None, + inference_response_keys=None, + ) + + assert isinstance(pipeline_model, PipelineModel) + + +def test_attach(sagemaker_session): + aml = AutoML.attach(auto_ml_job_name=JOB_NAME_3, sagemaker_session=sagemaker_session) + assert aml.current_job_name == JOB_NAME_3 + assert aml.role == "mock_role_arn" + assert aml.target_attribute_name == "y" + assert aml.problem_type == "Auto" + assert aml.output_path == "s3://output_prefix" + assert aml.tags == LIST_TAGS_RESULT["Tags"] From 42497966f709f95a57d4950abcb2077be6ea9550 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 12 Jun 2020 18:51:25 +0000 Subject: [PATCH 15/18] prepare release v1.63.0 --- CHANGELOG.md | 11 +++++++++++ VERSION | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6070d3862..b30a37ca62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v1.63.0 (2020-06-12) + +### Features + + * Allow selecting inference response content for automl generated models + * Support for multi variant endpoint invocation with target variant param + +### Documentation Changes + + * improve docstring and remove unavailable links + ## v1.62.0 (2020-06-11) ### Features diff --git a/VERSION b/VERSION index 69f7b1f2a8..af92bdd9f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.62.1.dev0 +1.63.0 From f586b3f5f51e99263ece3f284173014f635c2ce1 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 12 Jun 2020 19:22:50 +0000 Subject: [PATCH 16/18] update development version to v1.63.1.dev0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index af92bdd9f5..08819c9309 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.63.0 +1.63.1.dev0 From b2140785b81dc80a5119d85f6f576823596be3c4 Mon Sep 17 00:00:00 2001 From: Edward J Kim Date: Fri, 12 Jun 2020 16:56:50 -0700 Subject: [PATCH 17/18] feature: add support for SKLearn 0.23 (#1561) Co-authored-by: Chuyang --- src/sagemaker/fw_utils.py | 10 +++++++++- src/sagemaker/sklearn/defaults.py | 5 +++++ src/sagemaker/sklearn/estimator.py | 17 ++++++++++++----- tests/unit/test_sklearn.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/sagemaker/fw_utils.py b/src/sagemaker/fw_utils.py index 9141ae8c72..75a6ccffdb 100644 --- a/src/sagemaker/fw_utils.py +++ b/src/sagemaker/fw_utils.py @@ -587,10 +587,18 @@ def empty_framework_version_warning(default_version, latest_version): """ msgs = [EMPTY_FRAMEWORK_VERSION_WARNING.format(default_version)] if default_version != latest_version: - msgs.append(LATER_FRAMEWORK_VERSION_WARNING.format(latest=latest_version)) + msgs.append(later_framework_version_warning(latest_version)) return " ".join(msgs) +def later_framework_version_warning(latest_version): + """ + Args: + latest_version: + """ + return LATER_FRAMEWORK_VERSION_WARNING.format(latest=latest_version) + + def warn_if_parameter_server_with_multi_gpu(training_instance_type, distributions): """Warn the user that training will not fully leverage all the GPU cores if parameter server is enabled and a multi-GPU instance is selected. diff --git a/src/sagemaker/sklearn/defaults.py b/src/sagemaker/sklearn/defaults.py index 018611786a..34f5a3d2ef 100644 --- a/src/sagemaker/sklearn/defaults.py +++ b/src/sagemaker/sklearn/defaults.py @@ -15,6 +15,11 @@ SKLEARN_NAME = "scikit-learn" +# Default SKLearn version for when the framework version is not specified. +# This is no longer updated so as to not break existing workflows. SKLEARN_VERSION = "0.20.0" +SKLEARN_LATEST_VERSION = "0.23-1" +SKLEARN_SUPPORTED_VERSIONS = [SKLEARN_VERSION, SKLEARN_LATEST_VERSION] + LATEST_PY2_VERSION = "0.20.0" diff --git a/src/sagemaker/sklearn/estimator.py b/src/sagemaker/sklearn/estimator.py index 4e6a958d6b..f13e300427 100644 --- a/src/sagemaker/sklearn/estimator.py +++ b/src/sagemaker/sklearn/estimator.py @@ -19,7 +19,8 @@ from sagemaker.fw_registry import default_framework_uri from sagemaker.fw_utils import ( framework_name_from_image, - empty_framework_version_warning, + get_unsupported_framework_version_error, + later_framework_version_warning, python_deprecation_warning, ) from sagemaker.sklearn import defaults @@ -126,11 +127,17 @@ def __init__( self.py_version = py_version - if framework_version is None: - logger.warning( - empty_framework_version_warning(defaults.SKLEARN_VERSION, defaults.SKLEARN_VERSION) + if framework_version in defaults.SKLEARN_SUPPORTED_VERSIONS: + self.framework_version = framework_version + else: + raise ValueError( + get_unsupported_framework_version_error( + self.__framework_name__, framework_version, defaults.SKLEARN_SUPPORTED_VERSIONS + ) ) - self.framework_version = framework_version or defaults.SKLEARN_VERSION + + if framework_version != defaults.SKLEARN_LATEST_VERSION: + logger.warning(later_framework_version_warning(defaults.SKLEARN_LATEST_VERSION)) if image_name is None: image_tag = "{}-{}-{}".format(framework_version, "cpu", py_version) diff --git a/tests/unit/test_sklearn.py b/tests/unit/test_sklearn.py index 3acba6b8ee..ddc22230b6 100644 --- a/tests/unit/test_sklearn.py +++ b/tests/unit/test_sklearn.py @@ -574,6 +574,35 @@ def test_estimator_py2_warning(warning, sagemaker_session): warning.assert_called_with(estimator.__framework_name__, defaults.LATEST_PY2_VERSION) +@patch("sagemaker.sklearn.estimator.later_framework_version_warning") +def test_estimator_later_framework_version_warning(warning, sagemaker_session): + estimator = SKLearn( + entry_point=SCRIPT_PATH, + role=ROLE, + sagemaker_session=sagemaker_session, + train_instance_count=INSTANCE_COUNT, + train_instance_type=INSTANCE_TYPE, + ) + + assert estimator.framework_version == defaults.SKLEARN_VERSION + warning.assert_called_with(defaults.SKLEARN_LATEST_VERSION) + + +@patch("sagemaker.sklearn.estimator.get_unsupported_framework_version_error") +def test_estimator_throws_error_for_unsupported_version(error, sagemaker_session): + with pytest.raises(ValueError): + estimator = SKLearn( + entry_point=SCRIPT_PATH, + role=ROLE, + sagemaker_session=sagemaker_session, + train_instance_count=INSTANCE_COUNT, + train_instance_type=INSTANCE_TYPE, + framework_version="foo", + ) + assert estimator.framework_version not in defaults.SKLEARN_SUPPORTED_VERSIONS + error.assert_called_with(defaults.SKLEARN_NAME, "foo", defaults.SKLEARN_SUPPORT_VERSIONS) + + @patch("sagemaker.sklearn.model.python_deprecation_warning") def test_model_py2_warning(warning, sagemaker_session): source_dir = "s3://mybucket/source" From 0001e1b1eec58b611159310df40b423c9e4125ea Mon Sep 17 00:00:00 2001 From: Eric Johnson <65414824+metrizable@users.noreply.github.com> Date: Wed, 3 Jun 2020 11:19:42 -0700 Subject: [PATCH 18/18] change: include py38 tox env and some dependency upgrades --- .pylintrc | 19 ++++++++++--------- setup.py | 6 +++--- src/sagemaker/amazon/common.py | 4 ++-- src/sagemaker/cli/common.py | 2 +- src/sagemaker/rl/estimator.py | 12 +++++------- src/sagemaker/tuner.py | 6 +++--- src/sagemaker/workflow/airflow.py | 14 ++++++++------ tests/integ/test_monitoring_files.py | 6 +++--- tests/integ/test_multi_variant_endpoint.py | 4 ++-- tests/integ/test_tfs.py | 2 +- tests/unit/test_image.py | 4 ++-- tests/unit/test_tuner.py | 2 +- tox.ini | 8 ++++---- 13 files changed, 45 insertions(+), 44 deletions(-) diff --git a/.pylintrc b/.pylintrc index a0fbc2e148..e099e8f37b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -76,21 +76,22 @@ confidence= # --disable=W" disable= C0330, # Black disagrees with and explicitly violates this: https://github.com/python/black/issues/48 - too-many-locals, + abstract-method, # TODO: Fix abstract methods arguments-differ, - too-many-lines, + cyclic-import, # TODO: Resolve cyclic imports fixme, - too-many-arguments, invalid-name, - too-many-instance-attributes, - len-as-condition, # TODO: Enable this check once pylint 2.4.0 is released and consumed due to the fix in https://github.com/PyCQA/pylint/issues/2684 import-error, # Since we run Pylint before any of our builds in tox, this will always fail - protected-access, # TODO: Fix access - abstract-method, # TODO: Fix abstract methods - useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported. - cyclic-import, # TODO: Resolve cyclic imports + import-outside-toplevel, no-self-use, # TODO: Convert methods to functions where appropriate + protected-access, # TODO: Fix access + signature-differs, # TODO: fix kwargs + too-many-arguments, too-many-branches, # TODO: Simplify or ignore as appropriate + too-many-instance-attributes, + too-many-lines, + too-many-locals, + useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported. [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs diff --git a/setup.py b/setup.py index 8bb6525e97..14ecc4d7af 100644 --- a/setup.py +++ b/setup.py @@ -60,16 +60,16 @@ def read_version(): extras["test"] = ( [ extras["all"], - "tox==3.13.1", + "tox==3.15.1", "flake8", - "pytest==4.4.1", + "pytest==4.6.10", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "mock", "contextlib2", "awslogs", - "black==19.3b0 ; python_version >= '3.6'", + "black==19.10b0 ; python_version >= '3.6'", "stopit==1.1.2", "apache-airflow==1.10.5", "fabric>=2.0", diff --git a/src/sagemaker/amazon/common.py b/src/sagemaker/amazon/common.py index 1346e7059e..2be29e7a33 100644 --- a/src/sagemaker/amazon/common.py +++ b/src/sagemaker/amazon/common.py @@ -261,11 +261,11 @@ def read_recordio(f): """ while True: try: - read_kmagic, = struct.unpack("I", f.read(4)) + (read_kmagic,) = struct.unpack("I", f.read(4)) except struct.error: return assert read_kmagic == _kmagic - len_record, = struct.unpack("I", f.read(4)) + (len_record,) = struct.unpack("I", f.read(4)) pad = (((len_record + 3) >> 2) << 2) - len_record yield f.read(len_record) if pad: diff --git a/src/sagemaker/cli/common.py b/src/sagemaker/cli/common.py index 660fed64f5..7256937bc7 100644 --- a/src/sagemaker/cli/common.py +++ b/src/sagemaker/cli/common.py @@ -41,7 +41,7 @@ def __init__(self, args): self.script = args.script self.instance_type = args.instance_type self.instance_count = args.instance_count - self.environment = {k: v for k, v in (kv.split("=") for kv in args.env)} + self.environment = dict((kv.split("=") for kv in args.env)) self.session = sagemaker.Session() diff --git a/src/sagemaker/rl/estimator.py b/src/sagemaker/rl/estimator.py index a065bafb07..60c67ef5de 100644 --- a/src/sagemaker/rl/estimator.py +++ b/src/sagemaker/rl/estimator.py @@ -362,10 +362,10 @@ def _validate_framework_format(cls, framework): Args: framework: """ - if framework and framework not in RLFramework: + if framework and framework not in list(RLFramework): raise ValueError( - "Invalid type: {}, valid RL frameworks types are: [{}]".format( - framework, [t for t in RLFramework] + "Invalid type: {}, valid RL frameworks types are: {}".format( + framework, list(RLFramework) ) ) @@ -375,11 +375,9 @@ def _validate_toolkit_format(cls, toolkit): Args: toolkit: """ - if toolkit and toolkit not in RLToolkit: + if toolkit and toolkit not in list(RLToolkit): raise ValueError( - "Invalid type: {}, valid RL toolkits types are: [{}]".format( - toolkit, [t for t in RLToolkit] - ) + "Invalid type: {}, valid RL toolkits types are: {}".format(toolkit, list(RLToolkit)) ) @classmethod diff --git a/src/sagemaker/tuner.py b/src/sagemaker/tuner.py index 164b06ae71..7a74a95e24 100644 --- a/src/sagemaker/tuner.py +++ b/src/sagemaker/tuner.py @@ -92,10 +92,10 @@ def __init__(self, warm_start_type, parents): warm start the new tuning job. """ - if warm_start_type not in WarmStartTypes: + if warm_start_type not in list(WarmStartTypes): raise ValueError( - "Invalid type: {}, valid warm start types are: [{}]".format( - warm_start_type, [t for t in WarmStartTypes] + "Invalid type: {}, valid warm start types are: {}".format( + warm_start_type, list(WarmStartTypes) ) ) diff --git a/src/sagemaker/workflow/airflow.py b/src/sagemaker/workflow/airflow.py index 54ff0ee5df..bf3be24d5d 100644 --- a/src/sagemaker/workflow/airflow.py +++ b/src/sagemaker/workflow/airflow.py @@ -315,15 +315,17 @@ def tuning_config(tuner, inputs, job_name=None, include_cls_metadata=False, mini } if tuner.estimator: - tune_config[ - "TrainingJobDefinition" - ], s3_operations = _extract_training_config_from_estimator( + ( + tune_config["TrainingJobDefinition"], + s3_operations, + ) = _extract_training_config_from_estimator( tuner, inputs, include_cls_metadata, mini_batch_size ) else: - tune_config[ - "TrainingJobDefinitions" - ], s3_operations = _extract_training_config_list_from_estimator_dict( + ( + tune_config["TrainingJobDefinitions"], + s3_operations, + ) = _extract_training_config_list_from_estimator_dict( tuner, inputs, include_cls_metadata, mini_batch_size ) diff --git a/tests/integ/test_monitoring_files.py b/tests/integ/test_monitoring_files.py index d91042c394..c4be7eb017 100644 --- a/tests/integ/test_monitoring_files.py +++ b/tests/integ/test_monitoring_files.py @@ -319,7 +319,7 @@ def test_constraint_violations_object_creation_from_file_path_with_customization def test_constraint_violations_object_creation_from_file_path_without_customizations( - sagemaker_session + sagemaker_session, ): constraint_violations = ConstraintViolations.from_file_path( constraint_violations_file_path=os.path.join( @@ -354,7 +354,7 @@ def test_constraint_violations_object_creation_from_string_with_customizations( def test_constraint_violations_object_creation_from_string_without_customizations( - sagemaker_session + sagemaker_session, ): with open(os.path.join(tests.integ.DATA_DIR, "monitor/constraint_violations.json"), "r") as f: file_body = f.read() @@ -404,7 +404,7 @@ def test_constraint_violations_object_creation_from_s3_uri_with_customizations( def test_constraint_violations_object_creation_from_s3_uri_without_customizations( - sagemaker_session + sagemaker_session, ): with open(os.path.join(tests.integ.DATA_DIR, "monitor/constraint_violations.json"), "r") as f: file_body = f.read() diff --git a/tests/integ/test_multi_variant_endpoint.py b/tests/integ/test_multi_variant_endpoint.py index eddcc11768..83c57c7776 100644 --- a/tests/integ/test_multi_variant_endpoint.py +++ b/tests/integ/test_multi_variant_endpoint.py @@ -53,8 +53,8 @@ tests.integ.DATA_DIR, "sparkml_model", "mleap_model.tar.gz" ) SPARK_ML_DEFAULT_VARIANT_NAME = ( - "AllTraffic" -) # default defined in src/sagemaker/session.py def production_variant + "AllTraffic" # default defined in src/sagemaker/session.py def production_variant +) SPARK_ML_WRONG_VARIANT_NAME = "WRONG_VARIANT" SPARK_ML_TEST_DATA = "1.0,C,38.0,71.5,1.0,female" SPARK_ML_MODEL_SCHEMA = json.dumps( diff --git a/tests/integ/test_tfs.py b/tests/integ/test_tfs.py index 00fb6fdb75..f27b76a4f0 100644 --- a/tests/integ/test_tfs.py +++ b/tests/integ/test_tfs.py @@ -163,7 +163,7 @@ def test_predict_with_entry_point(tfs_predictor_with_model_and_entry_point_same_ @pytest.mark.local_mode def test_predict_with_model_and_entry_point_and_dependencies_separated( - tfs_predictor_with_model_and_entry_point_and_dependencies + tfs_predictor_with_model_and_entry_point_and_dependencies, ): input_data = {"instances": [1.0, 2.0, 5.0]} expected_result = {"predictions": [4.0, 4.5, 6.0]} diff --git a/tests/unit/test_image.py b/tests/unit/test_image.py index 9c5ecdbdbd..cf17441fea 100644 --- a/tests/unit/test_image.py +++ b/tests/unit/test_image.py @@ -799,7 +799,7 @@ def test__aws_credentials_with_long_lived_credentials(): @patch("sagemaker.local.image._aws_credentials_available_in_metadata_service") def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_having_credentials( - mock + mock, ): credentials = Credentials( access_key=_random_string(), secret_key=_random_string(), token=_random_string() @@ -814,7 +814,7 @@ def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_ @patch("sagemaker.local.image._aws_credentials_available_in_metadata_service") def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_having_no_credentials( - mock + mock, ): credentials = Credentials( access_key=_random_string(), secret_key=_random_string(), token=_random_string() diff --git a/tests/unit/test_tuner.py b/tests/unit/test_tuner.py index 490ae96145..67de14bc50 100644 --- a/tests/unit/test_tuner.py +++ b/tests/unit/test_tuner.py @@ -517,7 +517,7 @@ def test_attach_tuning_job_with_estimator_from_hyperparameters(sagemaker_session def test_attach_tuning_job_with_estimator_from_hyperparameters_with_early_stopping( - sagemaker_session + sagemaker_session, ): job_details = copy.deepcopy(TUNING_JOB_DETAILS) job_details["HyperParameterTuningJobConfig"]["TrainingJobEarlyStoppingType"] = "Auto" diff --git a/tox.ini b/tox.ini index 78cd0f1d4a..a376b68e4e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36,py37 +envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36,py37,py38 skip_missing_interpreters = False @@ -80,7 +80,7 @@ basepython = python3 skipdist = true skip_install = true deps = - pylint==2.3.1 + pylint==2.5.2 commands = python -m pylint --rcfile=.pylintrc -j 0 src/sagemaker @@ -129,13 +129,13 @@ commands = doc8 [testenv:black-format] # Used during development (before committing) to format .py files. basepython = python3 -deps = black==19.3b0 +deps = black==19.10b0 commands = black -l 100 ./ [testenv:black-check] # Used by automated build steps to check that all files are properly formatted. basepython = python3 -deps = black==19.3b0 +deps = black==19.10b0 commands = black -l 100 --check ./