aws · davidthomas426 · Jan 4, 2023 · Jan 4, 2023 · Jan 4, 2023 · Jan 4, 2023
diff --git a/README.rst b/README.rst
@@ -16,6 +16,22 @@ For information on running PyTorch jobs on Amazon SageMaker, please refer to the
 For notebook examples: `SageMaker Notebook
 Examples <https://github.com/awslabs/amazon-sagemaker-examples>`__.
 
+TorchServe Compatibility Requirements
+-------------------------------------
+
+Check this table for compatibility constraints.
+If version of sagemaker-pytorch-inference matches a row below, then the torchserve must also satisfy that row's constraint.
+
++---------------------------------+--------------------------+
+| ``sagemaker-pytorch-inference`` | ``torchserve``           |
++=================================+==========================+
+| ``main`` / ``nightly``          | ``main`` / ``nightly``   |
++---------------------------------+--------------------------+
+| ``>=2.0.9``                     | ``>=0.6``                |
++---------------------------------+--------------------------+
+| ``>=2.0.8``                     | ``>=0.5.3``              |
++---------------------------------+--------------------------+
+
 Contributing
 ------------
 

diff --git a/buildspec-release.yml b/buildspec-release.yml
@@ -10,16 +10,17 @@ phases:
       - tox -e flake8,twine
 
       # run unit tests
+      # TODO change below to `tox -e `py38,py39,py310 -- test/unit` once py39 and py310 are available
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py37 -- test/unit
+        tox -e py38 -- test/unit
 
       # run local integ tests
       #- $(aws ecr get-login --no-include-email --region us-west-2)
-      #- IGNORE_COVERAGE=- tox -e py27,py36 -- test/integ/local
+      #- IGNORE_COVERAGE=- tox -e py38,py39,py310 -- test/integ/local
 
       # run sagemaker integ tests
-      #- IGNORE_COVERAGE=- tox -e py27,py36 -- test/integ/sagemaker
+      #- IGNORE_COVERAGE=- tox -e py38,py39,py310 -- test/integ/sagemaker
 
       # generate the distribution package
       - python3 setup.py sdist

diff --git a/buildspec.yml b/buildspec.yml
@@ -2,7 +2,7 @@ version: 0.2
 
 env:
   variables:
-    FRAMEWORK_VERSION: '1.10.2'
+    FRAMEWORK_VERSION: '1.13.1'
     EIA_FRAMEWORK_VERSION: '1.3.1'
     CPU_INSTANCE_TYPE: 'ml.c4.xlarge'
     GPU_INSTANCE_TYPE: 'ml.p3.8xlarge'
@@ -11,7 +11,7 @@ env:
     GITHUB_REPO: 'sagemaker-pytorch-serving-container'
     DLC_ACCOUNT: '763104351884'
     SETUP_FILE: 'setup_cmds.sh'
-    SETUP_CMDS: '#!/bin/bash\npython3.6 -m pip install --upgrade pip\npython3.6 -m pip install -U -e .\npython3.6 -m pip install -U -e .[test]'
+    SETUP_CMDS: '#!/bin/bash\npython3.8 -m pip install --upgrade pip\npython3.8 -m pip install -U -e .\npython3.8 -m pip install -U -e .[test]'
 
 
 phases:
@@ -33,7 +33,8 @@ phases:
       - tox -e flake8,twine
 
       # run unit tests
-      - tox -e py36,py37 test/unit
+      # TODO change below to `tox -e `py38,py39,py310 -- test/unit` once py39 and py310 are available
+      - tox -e py38 -- test/unit
 
       # define tags
       - GENERIC_TAG="$FRAMEWORK_VERSION-pytorch-$BUILD_ID"
@@ -42,9 +43,9 @@ phases:
       - DLC_EIA_TAG="$EIA_FRAMEWORK_VERSION-dlc-eia-$BUILD_ID"
 
       # run local CPU integration tests (build and push the image to ECR repo)
-      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/local -vv -rA -s --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
-      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/local -vv -rA -s --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
 
       # launch remote GPU instance
@@ -65,27 +66,27 @@ phases:
       # run GPU local integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
       # no reason to rebuild the image again since it was already built and pushed to ECR during CPU tests
-      - generic_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
+      - generic_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/local -vv -rA -s --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
       - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
-      - dlc_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
+      - dlc_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/local -vv -rA -s --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
       - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\" --skip-setup"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
 
       # run CPU sagemaker integration tests
-      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GENERIC_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
-      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_CPU_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_CPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
 
       # run GPU sagemaker integration tests
-      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GENERIC_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
-      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $DLC_GPU_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $DLC_GPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
 
       # run EIA sagemaker integration tests
-      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --build-image --push-image --dockerfile-type dlc.eia --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $DLC_EIA_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py38 -- test/integration/sagemaker --build-image --push-image --dockerfile-type dlc.eia --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $DLC_EIA_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
 
     finally:

diff --git a/setup.py b/setup.py
@@ -39,25 +39,27 @@ def read(fname):
     url='https://github.com/aws/sagemaker-pytorch-inference-toolkit',
     license='Apache License 2.0',
 
+    python_requires=">=3.8",
+
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
         "Natural Language :: English",
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python",
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ],
 
     # We don't declare our dependency on torch here because we build with
     # different packages for different variants
-    install_requires=['numpy', 'retrying', 'sagemaker-inference>=1.3.1'],
+    install_requires=['numpy', 'retrying', 'sagemaker-inference>=1.8'],
     extras_require={
-        'test': ['boto3>=1.10.44', 'coverage==4.5.3', 'docker-compose==1.23.2', 'flake8==3.7.7', 'Flask==1.1.1',
-                 'mock==2.0.0', 'pytest==4.4.0', 'pytest-cov==2.7.1', 'pytest-xdist==1.28.0', 'PyYAML==3.10',
-                 'sagemaker==1.56.3', 'sagemaker-containers>=2.5.4', 'six==1.12.0', 'requests==2.20.0',
-                 'requests_mock==1.6.0', 'torch==1.6.0', 'torchvision==0.7.0', 'tox==3.7.0']
+        'test': ['boto3', 'coverage', 'docker-compose', 'flake8', 'Flask',
+                 'mock', 'pytest', 'pytest-cov', 'pytest-xdist', 'PyYAML',
+                 'sagemaker', 'six', 'requests',
+                 'requests_mock', 'torch==1.13.1', 'torchvision==0.14.1', 'tox']
     },
 
     entry_points={

diff --git a/test/container/1.13.1/Dockerfile.dlc.cpu b/test/container/1.13.1/Dockerfile.dlc.cpu
@@ -0,0 +1,9 @@
+ARG region
+FROM 763104351884.dkr.ecr.$region.amazonaws.com/pytorch-inference:1.13.1-cpu-py39-ubuntu20.04-sagemaker
+
+RUN pip uninstall torchserve -y && \
+    pip install torchserve-nightly==2022.3.23.post2
+
+COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz
+RUN pip install --upgrade --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \
+    rm /sagemaker_pytorch_inference.tar.gz
diff --git a/test/container/1.13.1/Dockerfile.dlc.gpu b/test/container/1.13.1/Dockerfile.dlc.gpu
@@ -0,0 +1,9 @@
+ARG region
+FROM 763104351884.dkr.ecr.$region.amazonaws.com/pytorch-inference:1.13.1-gpu-py38-cu117-ubuntu20.04-sagemaker
+
+RUN pip uninstall torchserve -y && \
+    pip install torchserve-nightly==2022.3.23.post2
+
+COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz
+RUN pip install --upgrade --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \
+    rm /sagemaker_pytorch_inference.tar.gz
diff --git a/test/container/1.13.1/Dockerfile.pytorch b/test/container/1.13.1/Dockerfile.pytorch
@@ -0,0 +1,46 @@
+FROM pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime
+
+LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
+LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
+
+ARG TS_VERSION=0.7.0
+ARG TS_ARCHIVER_VERSION=0.7.0
+
+ENV SAGEMAKER_SERVING_MODULE sagemaker_pytorch_serving_container.serving:main
+ENV TEMP=/home/model-server/tmp
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends software-properties-common \
+    && add-apt-repository ppa:openjdk-r/ppa \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    openjdk-11-jdk-headless \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN conda install -c conda-forge opencv \
+    && ln -s /opt/conda/bin/pip /usr/local/bin/pip3
+
+RUN pip install torchserve-nightly==2023.1.31 \
+ && pip install torch-model-archiver==$TS_ARCHIVER_VERSION
+
+COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz
+RUN pip install --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \
+    rm /sagemaker_pytorch_inference.tar.gz
+
+RUN useradd -m model-server \
+    && mkdir -p /home/model-server/tmp \
+    && chown -R model-server /home/model-server
+
+COPY artifacts/ts-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
+COPY artifacts/config.properties /home/model-server
+
+RUN chmod +x /usr/local/bin/dockerd-entrypoint.py
+
+EXPOSE 8080 8081
+ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"]
+CMD ["torchserve", "--start", "--ts-config", "/home/model-server/config.properties", "--model-store", "/home/model-server/"]
diff --git a/test/integration/local/test_serving.py b/test/integration/local/test_serving.py
@@ -22,7 +22,7 @@
 from sagemaker.pytorch import PyTorchModel
 from sagemaker.predictor import BytesDeserializer, csv_deserializer, csv_serializer, \
     json_deserializer, json_serializer, npy_serializer, numpy_deserializer
-from sagemaker_containers.beta.framework import content_types
+from sagemaker_inference import content_types
 from torchvision import datasets, transforms
 
 from integration import training_dir, mnist_1d_script, model_cpu_tar, mnist_cpu_script, \
@@ -89,10 +89,10 @@ def test_serving_calls_model_fn_once(image_uri, sagemaker_local_session, instanc
 @contextmanager
 def _predictor(model_tar, script, image, sagemaker_local_session, instance_type,
                model_server_workers=None):
-    model = PyTorchModel('file://{}'.format(model_tar),
-                         ROLE,
-                         script,
-                         image=image,
+    model = PyTorchModel(model_data='file://{}'.format(model_tar),
+                         role=ROLE,
+                         entry_point=script,
+                         image_uri=image,
                          sagemaker_session=sagemaker_local_session,
                          model_server_workers=model_server_workers)
 

diff --git a/tox.ini b/tox.ini
@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = flake8,py36,py37
+envlist = flake8,py38,py39,py310
 skip_missing_interpreters = False
 
 [flake8]
@@ -21,15 +21,17 @@ exclude =
 max-complexity = 10
 ignore =
     C901,
-    E203,  # whitespace before ':': Black disagrees with and explicitly violates this.
+    # E203: whitespace before ':': Black disagrees with and explicitly violates this.
+    E203,
     FI10,
     FI12,
     FI13,
     FI14,
     FI15,
     FI16,
     FI17,
-    FI18,  # __future__ import "annotations" missing -> check only Python 3.7 compatible
+    # FI18: __future__ import "annotations" missing -> check only Python 3.7 compatible
+    FI18,
     FI50,
     FI51,
     FI52,
@@ -53,34 +55,17 @@ passenv =
 commands =
     coverage run --rcfile .coveragerc --source sagemaker_pytorch_serving_container -m pytest {posargs}
     {env:IGNORE_COVERAGE:} coverage report --fail-under=90
-
-deps =
-    coverage
-    pytest
-    pytest-cov
-    pytest-xdist
-    mock
-    requests == 2.20.0
-    urllib3 < 1.23, >= 1.21
-    sagemaker < 2
-    sagemaker-containers
-    torch
-    torchvision
-    retrying
-    six
-    future
-    pyyaml
-    protobuf == 3.19.6
+extras = test
 
 [testenv:flake8]
-basepython = python3
+basepython = python3.8
 deps =
     flake8
     flake8-future-import
 commands = flake8
 
 [testenv:twine]
-basepython = python3
+basepython = python3.8
 # https://github.com/pypa/twine/blob/master/docs/changelog.rst
 deps =
     twine>=1.12.0

diff --git a/version-compatibility.json b/version-compatibility.json
@@ -0,0 +1,12 @@
+{
+    "constraints": [
+        {
+            ">=2.0.9": {
+                "torchserve": ">=0.6"
+            },
+            "`>=2.0.8": {
+                "torchserve": ">=0.5.3"
+            }
+        }
+    ]
+}