Merge branch 'master' into feat/instance-specific-jumpstart-host-requirements

Author: evakravi

.gitignore

@@ -34,4 +34,4 @@ env/
 **/_repack_script_launcher.sh
 tests/data/**/_repack_model.py
 tests/data/experiment/sagemaker-dev-1.0.tar.gz
-src/sagemaker/serve/tmp_workspace
+src/sagemaker/serve/tmp_workspace

CHANGELOG.md

@@ -1,5 +1,80 @@
 # Changelog
 
+## v2.209.0 (2024-02-24)
+
+### Features
+
+ * ModelBuilder to fetch local schema when no SchemaBuilder present.
+ * AutoGluon 1.0.0 image_uris update
+
+### Bug Fixes and Other Changes
+
+ * skip pytorch training compiler integ test
+ * add fixes for tarfile extractall functionality PEP-721
+ * Fix telemetry image uri option logic for ModelBuilder
+ * Add telemetry metrics on usage of default images for ModelBuilder
+ * Fix error message typo
+ * Add "distribution" parameter into record_set
+
+## v2.208.0 (2024-02-15)
+
+### Features
+
+ * Telemetry metrics
+ * TGI 1.4.0
+ * Support collection type and target store for feature store ingestion.
+
+### Bug Fixes and Other Changes
+
+ * bump jinja2 to 3.1.3 in doc/requirments.txt
+ * chore(deps): bump jinja2 from 3.0.3 to 3.1.3 in /requirements/extras
+ * Fix dependabot alert in transformers package
+ * Bump Apache Airflow version to 2.8.0
+ * skip failing mxnet tests
+
+### Documentation Changes
+
+ * change order of pipelines topics
+ * Explain the ClarifyCheckStep and QualityCheckStep parameters
+ * fix the ClarifyCheckStep documentation to mention PDP
+
+## v2.207.1 (2024-02-06)
+
+### Bug Fixes and Other Changes
+
+ * Add PT 2.1 as a supported framework for the smdistributed distribution
+ * Enable private docker registry support for ModelBuilder
+ * HF PT 2.1 Image Configs
+
+### Documentation Changes
+
+ * add setup commands for documentation generation
+
+## v2.207.0 (2024-02-05)
+
+### Features
+
+ * Introduce HF Transformers to ModelBuilder
+ * retrieve jumpstart estimator and predictor without specifying model id (infer from tags)
+
+### Bug Fixes and Other Changes
+
+ * SMP PT upgrade to 2.1
+ * Fetch HF metadata only when explicit type is not selected
+ * relax upper bound for urllib dependency
+
+## v2.206.0 (2024-01-31)
+
+### Features
+
+ * Logic to detect hardware GPU count and aggregate GPU memory size in MiB
+
+### Bug Fixes and Other Changes
+
+ * fixed create monitoring schedule failing after validation error
+ * Support PipelineVariable for ModelQualityCheckConfig attributes
+ * TGI NeuronX 0.0.17
+
 ## v2.205.0 (2024-01-25)
 
 ### Features

CONTRIBUTING.md

@@ -227,6 +227,12 @@ For example, see the [Processing API reference](https://github.com/aws/sagemaker
 
 To build the Sphinx docs, run the following command in the `doc/` directory:
 
+```shell
+# Initial setup, only required for the first run
+pip install -r requirements.txt
+pip install -e ../
+```
+
 ```shell
 make html
 ```

MANIFEST.in

@@ -1,6 +1,7 @@
 recursive-include src/sagemaker *.py
 
 include src/sagemaker/image_uri_config/*.json
+include src/sagemaker/serve/schema/*.json
 include src/sagemaker/serve/requirements.txt
 recursive-include requirements *
 

VERSION

@@ -1 +1 @@
-2.205.1.dev0
+2.209.1.dev0

doc/api/prep_data/feature_store.rst

@@ -91,6 +91,10 @@ Inputs
     :members:
     :show-inheritance:
 
+.. autoclass:: sagemaker.feature_store.inputs.TtlDuration
+    :members:
+    :show-inheritance:
+
 .. autoclass:: sagemaker.feature_store.inputs.S3StorageConfig
     :members:
     :show-inheritance:

doc/requirements.txt

@@ -2,5 +2,5 @@ sphinx==5.1.1
 sphinx-rtd-theme==0.5.0
 docutils==0.15.2
 packaging==20.9
-jinja2<3.1
+jinja2==3.1.3
 schema==0.7.5

doc/workflows/index.rst

@@ -7,7 +7,7 @@ The SageMaker Python SDK supports managed training and inference for a variety o
 .. toctree::
     :maxdepth: 2
 
+    pipelines/index
     airflow/index
     step_functions/index
-    pipelines/index
     lineage/index

requirements/extras/test_requirements.txt

@@ -12,13 +12,13 @@ awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.7.3
+apache-airflow==2.8.1
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0
 requests==2.31.0
 sagemaker-experiments==0.1.35
-Jinja2==3.0.3
+Jinja2==3.1.3
 pyvis==0.2.1
 pandas>=1.3.5,<1.5
 scikit-learn==1.3.0
@@ -32,7 +32,7 @@ sagemaker-feature-store-pyspark-3.3
 # TODO find workaround
 xgboost>=1.6.2,<=1.7.6
 pillow>=10.0.1,<=11
-transformers==4.32.0
+transformers>=4.36.0
 sentencepiece==0.1.99
 # https://github.com/triton-inference-server/server/issues/6246
 tritonclient[http]<2.37.0

setup.py

@@ -64,7 +64,7 @@ def read_requirements(filename):
     "jsonschema",
     "platformdirs",
     "tblib>=1.7.0,<3",
-    "urllib3<1.27",
+    "urllib3>=1.26.8,<3.0.0",
     "requests",
     "docker",
     "tqdm",

src/sagemaker/amazon/amazon_estimator.py

@@ -269,7 +269,14 @@ def fit(
         if wait:
             self.latest_training_job.wait(logs=logs)
 
-    def record_set(self, train, labels=None, channel="train", encrypt=False):
+    def record_set(
+        self,
+        train,
+        labels=None,
+        channel="train",
+        encrypt=False,
+        distribution="ShardedByS3Key",
+    ):
         """Build a :class:`~RecordSet` from a numpy :class:`~ndarray` matrix and label vector.
 
         For the 2D ``ndarray`` ``train``, each row is converted to a
@@ -294,6 +301,8 @@ def record_set(self, train, labels=None, channel="train", encrypt=False):
                 should be assigned to.
             encrypt (bool): Specifies whether the objects uploaded to S3 are
                 encrypted on the server side using AES-256 (default: ``False``).
+            distribution (str): The SageMaker TrainingJob channel s3 data
+                distribution type (default: ``ShardedByS3Key``).
 
         Returns:
             RecordSet: A RecordSet referencing the encoded, uploading training
@@ -316,6 +325,7 @@ def record_set(self, train, labels=None, channel="train", encrypt=False):
             num_records=train.shape[0],
             feature_dim=train.shape[1],
             channel=channel,
+            distribution=distribution,
         )
 
     def _get_default_mini_batch_size(self, num_records: int):
@@ -343,6 +353,7 @@ def __init__(
         feature_dim: int,
         s3_data_type: Union[str, PipelineVariable] = "ManifestFile",
         channel: Union[str, PipelineVariable] = "train",
+        distribution: str = "ShardedByS3Key",
     ):
         """A collection of Amazon :class:~`Record` objects serialized and stored in S3.
 
@@ -358,12 +369,15 @@ def __init__(
                 single s3 manifest file, listing each s3 object to train on.
             channel (str or PipelineVariable): The SageMaker Training Job channel this RecordSet
                 should be bound to
+            distribution (str): The SageMaker TrainingJob S3 data distribution type.
+                Valid values: 'ShardedByS3Key', 'FullyReplicated'.
         """
         self.s3_data = s3_data
         self.feature_dim = feature_dim
         self.num_records = num_records
         self.s3_data_type = s3_data_type
         self.channel = channel
+        self.distribution = distribution
 
     def __repr__(self):
         """Return an unambiguous representation of this RecordSet"""
@@ -377,7 +391,7 @@ def data_channel(self):
     def records_s3_input(self):
         """Return a TrainingInput to represent the training data"""
         return TrainingInput(
-            self.s3_data, distribution="ShardedByS3Key", s3_data_type=self.s3_data_type
+            self.s3_data, distribution=self.distribution, s3_data_type=self.s3_data_type
         )
 
 

src/sagemaker/estimator.py

@@ -3285,7 +3285,6 @@ class Framework(EstimatorBase):
     UNSUPPORTED_DLC_IMAGE_FOR_SM_PARALLELISM = (
         "2.0.1-gpu-py310-cu121",
         "2.0-gpu-py310-cu121",
-        "2.1.0-gpu-py310",
     )
 
     def __init__(
@@ -3959,7 +3958,7 @@ def _distribution_configuration(self, distribution):
             for unsupported_image in Framework.UNSUPPORTED_DLC_IMAGE_FOR_SM_PARALLELISM:
                 if (
                     unsupported_image in img_uri and not torch_distributed_enabled
-                ):  # disabling DLC images with CUDA12
+                ):  # disabling DLC images without SMDataParallel or SMModelParallel
                     raise ValueError(
                         f"SMDistributed is currently incompatible with DLC image: {img_uri}. "
                         "(Could be due to CUDA version being greater than 11.)"