Merge branch 'master' into mask-creds-local-mode

Author: jmahlik

buildspec-slowtests.yml

@@ -0,0 +1,11 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+      - IGNORE_COVERAGE=-
+
+      # slow tests
+      - start_time=`date +%s`
+      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m slow_test -n 10 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
+      - ./ci-scripts/displaytime.sh 'py38 slow tests' $start_time

buildspec.yml

@@ -16,7 +16,7 @@ phases:
 
       - start_time=`date +%s`
       - |
-        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
+        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron and not slow_test\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
       - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time
 
   post_build:

src/sagemaker/analytics.py

@@ -43,6 +43,7 @@ class AnalyticsMetricsBase(with_metaclass(ABCMeta, object)):
     """
 
     def __init__(self):
+        """Initializes ``AnalyticsMetricsBase`` instance."""
         self._dataframe = None
 
     def export_csv(self, filename):

tests/integ/test_auto_ml.py

@@ -47,6 +47,7 @@
 }
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
     reason="AutoML is not supported in the region yet.",

tests/integ/test_clarify_model_monitor.py

@@ -285,6 +285,7 @@ def test_bias_monitor(sagemaker_session, scheduled_bias_monitor, endpoint_name,
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",
@@ -304,76 +305,6 @@ def test_run_bias_monitor(
     scheduled_bias_monitor.delete_monitoring_schedule()
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
-    reason="ModelMonitoring is not yet supported in this region.",
-)
-def test_run_bias_monitor_baseline(
-    sagemaker_session,
-    data_config,
-    model_config,
-    bias_config,
-    model_predicted_label_config,
-    endpoint_name,
-    ground_truth_input,
-    upload_actual_data,
-):
-    monitor = ModelBiasMonitor(
-        role=ROLE,
-        instance_count=INSTANCE_COUNT,
-        instance_type=INSTANCE_TYPE,
-        volume_size_in_gb=VOLUME_SIZE_IN_GB,
-        max_runtime_in_seconds=MAX_RUNTIME_IN_SECONDS,
-        sagemaker_session=sagemaker_session,
-        tags=TEST_TAGS,
-    )
-
-    baselining_job_name = utils.unique_name_from_base("bias-baselining-job")
-    print("Creating baselining job: {}".format(baselining_job_name))
-    monitor.suggest_baseline(
-        data_config=data_config,
-        bias_config=bias_config,
-        model_config=model_config,
-        model_predicted_label_config=model_predicted_label_config,
-        job_name=baselining_job_name,
-    )
-    assert (
-        monitor.latest_baselining_job_config.probability_threshold_attribute
-        == BIAS_PROBABILITY_THRESHOLD
-    )
-    monitoring_schedule_name = utils.unique_name_from_base("bias-suggest-baseline")
-    s3_uri_monitoring_output = os.path.join(
-        "s3://",
-        sagemaker_session.default_bucket(),
-        endpoint_name,
-        monitoring_schedule_name,
-        "monitor_output",
-    )
-    # Let's test if the schedule can pick up analysis_config from baselining job
-    monitor.create_monitoring_schedule(
-        output_s3_uri=s3_uri_monitoring_output,
-        monitor_schedule_name=monitoring_schedule_name,
-        endpoint_input=EndpointInput(
-            endpoint_name=endpoint_name,
-            destination=ENDPOINT_INPUT_LOCAL_PATH,
-            start_time_offset=START_TIME_OFFSET,
-            end_time_offset=END_TIME_OFFSET,
-        ),
-        ground_truth_input=ground_truth_input,
-        schedule_cron_expression=CRON,
-    )
-    _verify_execution_status(monitor)
-
-    _verify_bias_job_description(
-        sagemaker_session=sagemaker_session,
-        monitor=monitor,
-        endpoint_name=endpoint_name,
-        ground_truth_input=ground_truth_input,
-    )
-
-    monitor.delete_monitoring_schedule()
-
-
 @pytest.fixture
 def explainability_monitor(sagemaker_session):
     monitor = ModelExplainabilityMonitor(
@@ -463,6 +394,7 @@ def test_explainability_monitor(sagemaker_session, scheduled_explainability_moni
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",
@@ -485,58 +417,6 @@ def test_run_explainability_monitor(
     scheduled_explainability_monitor.delete_monitoring_schedule()
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
-    reason="ModelMonitoring is not yet supported in this region.",
-)
-def test_run_explainability_monitor_baseline(
-    sagemaker_session, shap_config, data_config, model_config, endpoint_name, upload_actual_data
-):
-    monitor = ModelExplainabilityMonitor(
-        role=ROLE,
-        instance_count=INSTANCE_COUNT,
-        instance_type=INSTANCE_TYPE,
-        volume_size_in_gb=VOLUME_SIZE_IN_GB,
-        max_runtime_in_seconds=MAX_RUNTIME_IN_SECONDS,
-        sagemaker_session=sagemaker_session,
-        tags=TEST_TAGS,
-    )
-
-    baselining_job_name = utils.unique_name_from_base("explainability-baselining-job")
-    print("Creating baselining job: {}".format(baselining_job_name))
-    monitor.suggest_baseline(
-        data_config=data_config,
-        explainability_config=shap_config,
-        model_config=model_config,
-        job_name=baselining_job_name,
-    )
-    monitoring_schedule_name = utils.unique_name_from_base("explainability-suggest-baseline")
-    s3_uri_monitoring_output = os.path.join(
-        "s3://",
-        sagemaker_session.default_bucket(),
-        endpoint_name,
-        monitoring_schedule_name,
-        "monitor_output",
-    )
-    # Let's test if the schedule can pick up analysis_config from baselining job
-    monitor.create_monitoring_schedule(
-        output_s3_uri=s3_uri_monitoring_output,
-        monitor_schedule_name=monitoring_schedule_name,
-        endpoint_input=endpoint_name,
-        schedule_cron_expression=CRON,
-    )
-
-    _verify_execution_status(monitor)
-
-    _verify_explainability_job_description(
-        sagemaker_session=sagemaker_session,
-        monitor=monitor,
-        endpoint_name=endpoint_name,
-    )
-
-    monitor.delete_monitoring_schedule()
-
-
 def _verify_monitoring_schedule(monitor, schedule_status, schedule_cron_expression=CRON):
     desc = monitor.describe_schedule()
     assert desc["MonitoringScheduleName"] == monitor.monitoring_schedule_name

tests/integ/test_inference_pipeline.py

@@ -149,6 +149,7 @@ def test_inference_pipeline_model_deploy(sagemaker_session, cpu_instance_type):
         assert "Could not find model" in str(exception.value)
 
 
+@pytest.mark.slow_test
 def test_inference_pipeline_model_deploy_and_update_endpoint(
     sagemaker_session, cpu_instance_type, alternative_cpu_instance_type
 ):

tests/integ/test_model_quality_monitor.py

@@ -254,6 +254,7 @@ def test_run_model_quality_monitor(
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",

tests/integ/test_multidatamodel.py

@@ -262,6 +262,7 @@ def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, s
         assert "Could not find endpoint" in str(exception.value)
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() != "us-east-2",
     reason="Pulling the base image is currently limited to us-east-2.",
@@ -481,6 +482,7 @@ def __rcf_training_job(
         return rcf_model
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() != "us-east-2",
     reason="Pulling the base image is currently limited to us-east-2.",

tests/integ/test_mxnet.py

@@ -83,6 +83,7 @@ def test_attach_deploy(mxnet_training_job, sagemaker_session, cpu_instance_type)
         assert result is not None
 
 
+@pytest.mark.slow_test
 def test_deploy_estimator_with_different_instance_types(
     mxnet_training_job,
     sagemaker_session,
@@ -258,6 +259,7 @@ def test_deploy_model_with_tags_and_kms(
         assert endpoint_config["KmsKeyId"] == kms_key_arn
 
 
+@pytest.mark.slow_test
 def test_deploy_model_and_update_endpoint(
     mxnet_training_job,
     sagemaker_session,

tests/integ/test_tuner.py

@@ -523,6 +523,7 @@ def test_stop_tuning_job(sagemaker_session, cpu_instance_type):
     assert desc["HyperParameterTuningJobStatus"] == "Stopping"
 
 
+@pytest.mark.slow_test
 @pytest.mark.canary_quick
 def test_tuning_mxnet(
     sagemaker_session,