Skip to content

Commit ee87eb0

Browse files
maaquibmaaquib
authored
Merge branch 'master' into patch-1
2 parents 35d0963 + 0513c7a commit ee87eb0

24 files changed

+738
-180
lines changed

‎CHANGELOG.md

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,35 @@
11
# Changelog
22

3+
## v2.0.10 (2022-04-07)
4+
5+
### Bug Fixes and Other Changes
6+
7+
* pass model directory as input to torchserve
8+
9+
## v2.0.9 (2022-04-04)
10+
11+
### Bug Fixes and Other Changes
12+
13+
* Update CI to use PyTorch 1.10
14+
15+
## v2.0.8 (2022-01-13)
16+
17+
### Bug Fixes and Other Changes
18+
19+
* log4j migration from 1 to 2. Replace properties file with xml.
20+
21+
## v2.0.7 (2021-10-26)
22+
23+
### Bug Fixes and Other Changes
24+
25+
* Enable default model fn for cpu and gpu
26+
27+
## v2.0.6 (2021-10-04)
28+
29+
### Bug Fixes and Other Changes
30+
31+
* Env variable support for batch inference
32+
333
## v2.0.5 (2021-03-17)
434

535
### Bug Fixes and Other Changes

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.0.6.dev0
1+
2.0.11.dev0

‎buildspec.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ version: 0.2
22

33
env:
44
variables:
5-
FRAMEWORK_VERSION: '1.6.0'
5+
FRAMEWORK_VERSION: '1.10.2'
66
EIA_FRAMEWORK_VERSION: '1.3.1'
77
CPU_INSTANCE_TYPE: 'ml.c4.xlarge'
8-
GPU_INSTANCE_TYPE: 'ml.p2.8xlarge'
8+
GPU_INSTANCE_TYPE: 'ml.p3.8xlarge'
99
EIA_ACCELERATOR_TYPE: 'ml.eia2.medium'
1010
ECR_REPO: 'sagemaker-test'
1111
GITHUB_REPO: 'sagemaker-pytorch-serving-container'
@@ -39,12 +39,12 @@ phases:
3939
- GENERIC_TAG="$FRAMEWORK_VERSION-pytorch-$BUILD_ID"
4040
- DLC_CPU_TAG="$FRAMEWORK_VERSION-dlc-cpu-$BUILD_ID"
4141
- DLC_GPU_TAG="$FRAMEWORK_VERSION-dlc-gpu-$BUILD_ID"
42-
- DLC_EIA_TAG="$FRAMEWORK_VERSION-dlc-eia-$BUILD_ID"
42+
- DLC_EIA_TAG="$EIA_FRAMEWORK_VERSION-dlc-eia-$BUILD_ID"
4343

4444
# run local CPU integration tests (build and push the image to ECR repo)
45-
- test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
45+
- test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
4646
- execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
47-
- test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
47+
- test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
4848
- execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
4949

5050
# launch remote GPU instance
@@ -65,10 +65,10 @@ phases:
6565
# run GPU local integration tests
6666
- printf "$SETUP_CMDS" > $SETUP_FILE
6767
# no reason to rebuild the image again since it was already built and pushed to ECR during CPU tests
68-
- generic_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
68+
- generic_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
6969
- test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
7070
- execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
71-
- dlc_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
71+
- dlc_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -vv -rA -s --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
7272
- test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\" --skip-setup"
7373
- execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "artifacts/*"
7474

‎src/sagemaker_pytorch_serving_container/default_pytorch_inference_handler.py

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
from __future__ import absolute_import
1414

1515
import os
16-
import textwrap
1716

1817
import torch
1918
from sagemaker_inference import (
@@ -29,9 +28,21 @@
2928
DEFAULT_MODEL_FILENAME = "model.pt"
3029

3130

31+
class ModelLoadError(Exception):
32+
pass
33+
34+
3235
class DefaultPytorchInferenceHandler(default_inference_handler.DefaultInferenceHandler):
3336
VALID_CONTENT_TYPES = (content_types.JSON, content_types.NPY)
3437

38+
@staticmethod
39+
def _is_model_file(filename):
40+
is_model_file = False
41+
if os.path.isfile(filename):
42+
_, ext = os.path.splitext(filename)
43+
is_model_file = ext in [".pt", ".pth"]
44+
return is_model_file
45+
3546
def default_model_fn(self, model_dir):
3647
"""Loads a model. For PyTorch, a default function to load a model only if Elastic Inference is used.
3748
In other cases, users should provide customized model_fn() in script.
@@ -47,12 +58,30 @@ def default_model_fn(self, model_dir):
4758
raise FileNotFoundError("Failed to load model with default model_fn: missing file {}."
4859
.format(DEFAULT_MODEL_FILENAME))
4960
# Client-framework is CPU only. But model will run in Elastic Inference server with CUDA.
50-
return torch.jit.load(model_path, map_location=torch.device('cpu'))
61+
try:
62+
return torch.jit.load(model_path, map_location=torch.device('cpu'))
63+
except RuntimeError as e:
64+
raise ModelLoadError(
65+
"Failed to load {}. Please ensure model is saved using torchscript.".format(model_path)
66+
) from e
5167
else:
52-
raise NotImplementedError(textwrap.dedent("""
53-
Please provide a model_fn implementation.
54-
See documentation for model_fn at https://github.com/aws/sagemaker-python-sdk
55-
"""))
68+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
69+
model_path = os.path.join(model_dir, DEFAULT_MODEL_FILENAME)
70+
if not os.path.exists(model_path):
71+
model_files = [file for file in os.listdir(model_dir) if self._is_model_file(file)]
72+
if len(model_files) != 1:
73+
raise ValueError(
74+
"Exactly one .pth or .pt file is required for PyTorch models: {}".format(model_files)
75+
)
76+
model_path = os.path.join(model_dir, model_files[0])
77+
try:
78+
model = torch.jit.load(model_path, map_location=device)
79+
except RuntimeError as e:
80+
raise ModelLoadError(
81+
"Failed to load {}. Please ensure model is saved using torchscript.".format(model_path)
82+
) from e
83+
model = model.to(device)
84+
return model
5685

5786
def default_input_fn(self, input_data, content_type):
5887
"""A default input_fn that can handle JSON, CSV and NPZ formats.

‎src/sagemaker_pytorch_serving_container/etc/log4j.properties

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎src/sagemaker_pytorch_serving_container/etc/log4j2.xml

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<Configuration>
3+
<Appenders>
4+
<Console name="STDOUT" target="SYSTEM_OUT">
5+
<PatternLayout pattern="%d{ISO8601} [%-5p] %t %c - %m%n"/>
6+
</Console>
7+
<RollingFile
8+
name="access_log"
9+
fileName="${env:LOG_LOCATION:-logs}/access_log.log"
10+
filePattern="${env:LOG_LOCATION:-logs}/access_log.%d{dd-MMM}.log.gz">
11+
<PatternLayout pattern="%d{ISO8601} - %m%n"/>
12+
<Policies>
13+
<SizeBasedTriggeringPolicy size="10 MB"/>
14+
<TimeBasedTriggeringPolicy/>
15+
</Policies>
16+
<DefaultRolloverStrategy max="5"/>
17+
</RollingFile>
18+
<RollingFile
19+
name="model_log"
20+
fileName="${env:LOG_LOCATION:-logs}/model_log.log"
21+
filePattern="${env:LOG_LOCATION:-logs}/model_log.%d{dd-MMM}.log.gz">
22+
<PatternLayout pattern="%d{ISO8601} [%-5p] %c - %m%n"/>
23+
<Policies>
24+
<SizeBasedTriggeringPolicy size="10 MB"/>
25+
<TimeBasedTriggeringPolicy/>
26+
</Policies>
27+
<DefaultRolloverStrategy max="5"/>
28+
</RollingFile>
29+
<RollingFile
30+
name="model_metrics"
31+
fileName="${env:METRICS_LOCATION:-logs}/model_metrics.log"
32+
filePattern="${env:METRICS_LOCATION:-logs}/model_metrics.%d{dd-MMM}.log.gz">
33+
<PatternLayout pattern="%d{ISO8601} - %m%n"/>
34+
<Policies>
35+
<SizeBasedTriggeringPolicy size="10 MB"/>
36+
<TimeBasedTriggeringPolicy/>
37+
</Policies>
38+
<DefaultRolloverStrategy max="5"/>
39+
</RollingFile>
40+
<RollingFile
41+
name="ts_log"
42+
fileName="${env:LOG_LOCATION:-logs}/ts_log.log"
43+
filePattern="${env:LOG_LOCATION:-logs}/ts_log.%d{dd-MMM}.log.gz">
44+
<PatternLayout pattern="%d{ISO8601} [%-5p] %t %c - %m%n"/>
45+
<Policies>
46+
<SizeBasedTriggeringPolicy size="10 MB"/>
47+
<TimeBasedTriggeringPolicy/>
48+
</Policies>
49+
<DefaultRolloverStrategy max="5"/>
50+
</RollingFile>
51+
<RollingFile
52+
name="ts_metrics"
53+
fileName="${env:METRICS_LOCATION:-logs}/ts_metrics.log"
54+
filePattern="${env:METRICS_LOCATION:-logs}/ts_metrics.%d{dd-MMM}.log.gz">
55+
<PatternLayout pattern="%d{ISO8601} - %m%n"/>
56+
<Policies>
57+
<SizeBasedTriggeringPolicy size="10 MB"/>
58+
<TimeBasedTriggeringPolicy/>
59+
</Policies>
60+
<DefaultRolloverStrategy max="5"/>
61+
</RollingFile>
62+
</Appenders>
63+
<Loggers>
64+
<Logger name="ACCESS_LOG" level="info">
65+
<AppenderRef ref="access_log"/>
66+
</Logger>
67+
<Logger name="io.netty" level="error"/>
68+
<Logger name="MODEL_LOG" level="info">
69+
<AppenderRef ref="model_log"/>
70+
</Logger>
71+
<Logger name="MODEL_METRICS" level="info">
72+
<AppenderRef ref="model_metrics"/>
73+
</Logger>
74+
<Logger name="org.apache" level="off"/>
75+
<Logger name="org.pytorch.serve" level="info">
76+
<AppenderRef ref="ts_log"/>
77+
</Logger>
78+
<Logger name="TS_METRICS" level="info">
79+
<AppenderRef ref="ts_metrics"/>
80+
</Logger>
81+
<Root level="info">
82+
<AppenderRef ref="STDOUT"/>
83+
<AppenderRef ref="ts_log"/>
84+
</Root>
85+
</Loggers>
86+
</Configuration>
87+

0 commit comments

Comments
 (0)