Merge branch 'master' into local-gpu

Author: chuyang-deng

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## v1.64.0 (2020-06-15)
+
+### Features
+
+ * add support for SKLearn 0.23
+
+## v1.63.0 (2020-06-12)
+
+### Features
+
+ * Allow selecting inference response content for automl generated models
+ * Support for multi variant endpoint invocation with target variant param
+
+### Documentation Changes
+
+ * improve docstring and remove unavailable links
+
 ## v1.62.0 (2020-06-11)
 
 ### Features

VERSION

@@ -1 +1 @@
-1.62.1.dev0
+1.64.1.dev0

src/sagemaker/automl/automl.py

@@ -587,10 +587,18 @@ def empty_framework_version_warning(default_version, latest_version):
     """
     msgs = [EMPTY_FRAMEWORK_VERSION_WARNING.format(default_version)]
     if default_version != latest_version:
-        msgs.append(LATER_FRAMEWORK_VERSION_WARNING.format(latest=latest_version))
+        msgs.append(later_framework_version_warning(latest_version))
     return " ".join(msgs)
 
 
+def later_framework_version_warning(latest_version):
+    """
+    Args:
+        latest_version:
+    """
+    return LATER_FRAMEWORK_VERSION_WARNING.format(latest=latest_version)
+
+
 def warn_if_parameter_server_with_multi_gpu(training_instance_type, distributions):
     """Warn the user that training will not fully leverage all the GPU
     cores if parameter server is enabled and a multi-GPU instance is selected.

src/sagemaker/fw_utils.py

@@ -15,6 +15,11 @@
 
 SKLEARN_NAME = "scikit-learn"
 
+# Default SKLearn version for when the framework version is not specified.
+# This is no longer updated so as to not break existing workflows.
 SKLEARN_VERSION = "0.20.0"
+SKLEARN_LATEST_VERSION = "0.23-1"
+SKLEARN_SUPPORTED_VERSIONS = [SKLEARN_VERSION, SKLEARN_LATEST_VERSION]
+
 
 LATEST_PY2_VERSION = "0.20.0"

src/sagemaker/sklearn/defaults.py

@@ -19,7 +19,8 @@
 from sagemaker.fw_registry import default_framework_uri
 from sagemaker.fw_utils import (
     framework_name_from_image,
-    empty_framework_version_warning,
+    get_unsupported_framework_version_error,
+    later_framework_version_warning,
     python_deprecation_warning,
 )
 from sagemaker.sklearn import defaults
@@ -126,11 +127,17 @@ def __init__(
 
         self.py_version = py_version
 
-        if framework_version is None:
-            logger.warning(
-                empty_framework_version_warning(defaults.SKLEARN_VERSION, defaults.SKLEARN_VERSION)
+        if framework_version in defaults.SKLEARN_SUPPORTED_VERSIONS:
+            self.framework_version = framework_version
+        else:
+            raise ValueError(
+                get_unsupported_framework_version_error(
+                    self.__framework_name__, framework_version, defaults.SKLEARN_SUPPORTED_VERSIONS
+                )
             )
-        self.framework_version = framework_version or defaults.SKLEARN_VERSION
+
+        if framework_version != defaults.SKLEARN_LATEST_VERSION:
+            logger.warning(later_framework_version_warning(defaults.SKLEARN_LATEST_VERSION))
 
         if image_name is None:
             image_tag = "{}-{}-{}".format(framework_version, "cpu", py_version)

src/sagemaker/sklearn/estimator.py

@@ -0,0 +1,15 @@
+6.4,2.8,5.6,2.2
+5.0,2.3,3.3,1.0
+4.9,2.5,4.5,1.7
+4.9,3.1,1.5,0.1
+5.7,3.8,1.7,0.3
+4.4,3.2,1.3,0.2
+5.4,3.4,1.5,0.4
+6.9,3.1,5.1,2.3
+6.7,3.1,4.4,1.4
+5.1,3.7,1.5,0.4
+5.2,2.7,3.9,1.4
+6.9,3.1,4.9,1.5
+5.8,4.0,1.2,0.2
+5.4,3.9,1.7,0.4
+7.7,3.8,6.7,2.2

tests/data/automl/data/iris_transform.csv

@@ -32,11 +32,14 @@
 DATA_DIR = os.path.join(DATA_DIR, "automl", "data")
 TRAINING_DATA = os.path.join(DATA_DIR, "iris_training.csv")
 TEST_DATA = os.path.join(DATA_DIR, "iris_test.csv")
+TRANSFORM_DATA = os.path.join(DATA_DIR, "iris_transform.csv")
 PROBLEM_TYPE = "MultiClassClassification"
 BASE_JOB_NAME = "auto-ml"
 
 # use a succeeded AutoML job to test describe and list candidates method, otherwise tests will run too long
 AUTO_ML_JOB_NAME = "python-sdk-integ-test-base-job"
+DEFAULT_MODEL_NAME = "python-sdk-automl"
+
 
 EXPECTED_DEFAULT_JOB_CONFIG = {
     "CompletionCriteria": {"MaxCandidates": 3},
@@ -180,6 +183,42 @@ def test_auto_ml_describe_auto_ml_job(sagemaker_session):
     assert desc["OutputDataConfig"] == expected_default_output_config
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
+def test_auto_ml_attach(sagemaker_session):
+    expected_default_input_config = [
+        {
+            "DataSource": {
+                "S3DataSource": {
+                    "S3DataType": "S3Prefix",
+                    "S3Uri": "s3://{}/{}/input/iris_training.csv".format(
+                        sagemaker_session.default_bucket(), PREFIX
+                    ),
+                }
+            },
+            "TargetAttributeName": TARGET_ATTRIBUTE_NAME,
+        }
+    ]
+    expected_default_output_config = {
+        "S3OutputPath": "s3://{}/".format(sagemaker_session.default_bucket())
+    }
+
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
+    attached_automl_job = AutoML.attach(
+        auto_ml_job_name=AUTO_ML_JOB_NAME, sagemaker_session=sagemaker_session
+    )
+    attached_desc = attached_automl_job.describe_auto_ml_job()
+    assert attached_desc["AutoMLJobName"] == AUTO_ML_JOB_NAME
+    assert attached_desc["AutoMLJobStatus"] == "Completed"
+    assert isinstance(attached_desc["BestCandidate"], dict)
+    assert attached_desc["InputDataConfig"] == expected_default_input_config
+    assert attached_desc["AutoMLJobConfig"] == EXPECTED_DEFAULT_JOB_CONFIG
+    assert attached_desc["OutputDataConfig"] == expected_default_output_config
+
+
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
     reason="AutoML is not supported in the region yet.",
@@ -240,6 +279,38 @@ def test_deploy_best_candidate(sagemaker_session, cpu_instance_type):
     sagemaker_session.sagemaker_client.delete_endpoint(EndpointName=endpoint_name)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
+def test_create_model_best_candidate(sagemaker_session, cpu_instance_type):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
+    auto_ml = AutoML.attach(auto_ml_job_name=AUTO_ML_JOB_NAME, sagemaker_session=sagemaker_session)
+    best_candidate = auto_ml.best_candidate()
+
+    with timeout(minutes=5):
+        pipeline_model = auto_ml.create_model(
+            name=DEFAULT_MODEL_NAME,
+            candidate=best_candidate,
+            sagemaker_session=sagemaker_session,
+            vpc_config=None,
+            enable_network_isolation=False,
+            model_kms_key=None,
+            predictor_cls=None,
+        )
+    inputs = sagemaker_session.upload_data(
+        path=TRANSFORM_DATA, key_prefix=PREFIX + "/transform_input"
+    )
+    pipeline_model.transformer(
+        instance_count=1,
+        instance_type=cpu_instance_type,
+        assemble_with="Line",
+        output_path="s3://{}/{}".format(sagemaker_session.default_bucket(), "transform_test"),
+        accept="text/csv",
+    ).transform(data=inputs, content_type="text/csv", split_type="Line", join_source="Input")
+
+
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
     reason="AutoML is not supported in the region yet.",