Merge branch 'master' into issue_1107

Author: lxning

.circleci/README.md

@@ -54,6 +54,7 @@ Following _executors_ are available for job execution -
 2. ubuntu18-conda38-cpu-docker
 3. ubuntu18-pyenv37-cpu-docker
 4. ubuntu18-venv36-cpu-docker
+5. ubuntu18-conda39-cpu-docker
 
 > Please check the _workflows_, _jobs_ and _executors_ section in _config.yml_ for an up to date list
 
@@ -74,6 +75,7 @@ Available environment types are
 * `conda38`: python 3.8 with `conda` env
 * `pyenv37`: python 3.7 with `pyenv` env
 * `venv36`: python 3.6.9 with `venv` env
+* `conda39`: python 3.9 with `conda` env
 ```
 ./build_cci_image.sh --env_type <env-type>
 ```

.circleci/config.yml

@@ -3,6 +3,9 @@ version: 2.1
 
 executors:
   # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY under 'Project Settings > Environment Variables' in CircleCI
+  ubuntu18-conda39-cpu-docker:
+    docker:
+      - image: 630887156731.dkr.ecr.us-east-1.amazonaws.com/torchserve-build:ubuntu18-conda39-cpu
   ubuntu18-pythn36-cpu-docker:
     docker:
       - image: 630887156731.dkr.ecr.us-east-1.amazonaws.com/torchserve-build:ubuntu18-pythn36-cpu
@@ -250,6 +253,7 @@ workflows:
           matrix: &matrix
             parameters:
               executor:
+                - ubuntu18-conda39-cpu-docker
                 - ubuntu18-pythn36-cpu-docker
                 - ubuntu18-conda38-cpu-docker
                 - ubuntu18-pyenv37-cpu-docker

.circleci/images/Dockerfile

@@ -19,6 +19,16 @@ RUN DEBIAN_FRONTEND=noninteractive \
     && apt-get install --no-install-recommends -y \
         nodejs
 
+# Install python 3.9 using conda env
+FROM compile-image as conda39-image
+ENV PATH ~/.local/bin/:$PATH
+ENV PATH=/miniconda/bin:${PATH}
+RUN curl -LO http://repo.continuum.io/miniconda/Miniconda3-py39_4.9.2-Linux-x86_64.sh \
+        && bash Miniconda3-py39_4.9.2-Linux-x86_64.sh -p /miniconda -b \
+        && rm Miniconda3-py39_4.9.2-Linux-x86_64.sh \
+        && rm -rf /var/lib/apt/lists/* \
+        && conda install -y conda-build
+
 # Install python 3.8 using conda env
 FROM compile-image as conda38-image
 ENV PATH ~/.local/bin/:$PATH
@@ -39,7 +49,8 @@ RUN apt-get install --no-install-recommends -y \
         python3-setuptools \
         python3-wheel \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
-    && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
+    && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 \
+    && pip3 install --upgrade pip 
 
 # Install python 3.7 using pyenv env
 FROM compile-image as pyenv37-image
@@ -50,7 +61,8 @@ RUN apt-get install --no-install-recommends -y  libssl-dev zlib1g-dev libbz2-dev
         libsqlite3-dev libxml2-dev libxmlsec1-dev libffi-dev\
     && curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash \
     && pyenv install 3.7.0 \
-    && pyenv global 3.7.0
+    && pyenv global 3.7.0 \
+    && pip3 install --upgrade pip
 
 # Install python 3.6 and venv env
 FROM compile-image as venv36-image
@@ -66,7 +78,8 @@ RUN apt-get install --no-install-recommends -y \
         python3-venv\
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1\
-    && python3 -m venv $VIRTUAL_ENV
+    && python3 -m venv $VIRTUAL_ENV \
+    && pip3 install --upgrade pip
 
 
 FROM ${ENV_TYPE}-image AS final-image
@@ -103,9 +116,9 @@ RUN pip install --no-cache-dir \
         pytest-mock \
         requests \
         sentencepiece \
-        torch==1.6.0 \
-        torchaudio==0.6.0 \
-        torchtext==0.7.0 \
-        torchvision==0.7.0 \
-        transformers\
-        wheel
+        torch==1.8.1 \
+        torchaudio==0.8.1 \
+        torchtext==0.9.1 \
+        torchvision==0.9.1 \
+        transformers==4.6.0 \
+        wheel

.circleci/images/build_cci_image.sh

@@ -9,7 +9,7 @@ do
         -h|--help)
           echo "options:"
           echo "-h, --help  show brief help"
-          echo "-e, --env_type specify env_type to use from { pythn36, conda38, pyenv37, venv36 }"
+          echo "-e, --env_type specify env_type to use from { pythn36, conda38, pyenv37, venv36, conda39 }"
           echo "-t, --tag specify tag name for docker image \"<image>:<tag>\""
           exit 0
           ;;

.circleci/scripts/linux_test_sanity.sh

@@ -2,7 +2,7 @@
 
 source ts_scripts/install_utils
 
-MODELS=("fastrcnn" "fcn_resnet_101" "deeplabv3_resnet_101_eager" "my_text_classifier_v2" "resnet-18" "my_text_classifier_scripted_v2" "alexnet_scripted" "fcn_resnet_101_scripted"
+MODELS=("fastrcnn" "fcn_resnet_101" "deeplabv3_resnet_101_eager" "my_text_classifier_v4" "resnet-18" "my_text_classifier_scripted_v3" "alexnet_scripted" "fcn_resnet_101_scripted"
            "deeplabv3_resnet_101_scripted" "distill_bert_qa_eager" "bert_token_classification_no_torchscript" "bert_seqc_without_torchscript")
 MODEL_INPUTS=("examples/object_detector/persons.jpg,docs/images/blank_image.jpg" "examples/image_segmenter/persons.jpg" "examples/image_segmenter/persons.jpg"
  "examples/text_classification/sample_text.txt" "examples/image_classifier/kitten.jpg" "examples/text_classification/sample_text.txt" "examples/image_classifier/kitten.jpg"

.github/workflows/doc-automation.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6]
+        python-version: [3.8]
     steps:
     - name: Checkout
       uses: actions/checkout@v2

.gitignore

@@ -10,6 +10,8 @@ dist/
 .github/.DS_Store
 .DS_Store
 frontend/server/src/main/java/org/pytorch/serve/grpc/
+*.pem
+*.backup
 
 # Postman files
 test/artifacts/
@@ -18,5 +20,10 @@ test/model_store/
 test/ts_console.log
 test/config.properties
 
+
 .vscode
 .scratch/
+
+# Custom benchmark artifacts
+instances.yaml
+instances.yaml.backup

CONTRIBUTING.md

@@ -21,11 +21,14 @@ If you are interested in contributing to TorchServe, your contributions will fal
            ```
             > Supported cuda versions as cu111, cu102, cu101, cu92
 
-        - Execute sanity suite
+        - Run sanity suite
           ```bash
-          python ./torchserve_sanity.py
+          python torchserve_sanity.py
           ```
+    - Run Regression test `python test/regression_tests.py`
     - For running individual test suites refer [code_coverage](docs/code_coverage.md) documentation
+    - If you are updating an existing model make sure that performance hasn't degraded by running [benchmarks](https://github.com/pytorch/serve/tree/master/benchmarks) on the master branch and your branch and verify there is no performance regression 
+    - For large changes make sure to run the [automated benchmark suite](https://github.com/pytorch/serve/tree/master/test/benchmark) which will run the apache bench tests on several configurations of CUDA and EC2 instances
     - If you need more context on a particular issue, please create raise a ticket on [`TorchServe` GH repo](https://github.com/pytorch/serve/issues/new/choose) or connect to [PyTorch's slack channel](https://pytorch.slack.com/)
 
 Once you finish implementing a feature or bug-fix, please send a Pull Request to https://github.com/pytorch/serve. Use this [template](pull_request_template.md) when creating a Pull Request.

README.md

@@ -22,6 +22,8 @@ TorchServe is a flexible and easy to use tool for serving PyTorch models.
 * [Serve a Model](#serve-a-model)
 * [Serve a Workflow](docs/workflows.md)
 * [Quick start with docker](#quick-start-with-docker)
+* [Highlighted Examples](#highlighted-examples)
+* [Featured Community Projects](#featured-community-projects)
 * [Contributing](#contributing)
 
 ## Install TorchServe and torch-model-archiver
@@ -43,22 +45,24 @@ TorchServe is a flexible and easy to use tool for serving PyTorch models.
        ```bash
        python ./ts_scripts/install_dependencies.py --cuda=cu102
        ```
+       
+     Note: PyTorch 1.9+ will not support cu92 and cu101. So TorchServe only supports cu92 and cu101 up to PyTorch 1.8.1.  
 
     #### For Windows
 
     Refer to the documentation [here](docs/torchserve_on_win_native.md).
 
-2. Install torchserve and torch-model-archiver
+2. Install torchserve, torch-model-archiver and torch-workflow-archiver
 
     For [Conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install)  
     Note: Conda packages are not supported for Windows. Refer to the documentation [here](docs/torchserve_on_win_native.md).
     ```
-    conda install torchserve torch-model-archiver -c pytorch
+    conda install torchserve torch-model-archiver torch-workflow-archiver -c pytorch
     ```
 
     For Pip
     ```
-    pip install torchserve torch-model-archiver
+    pip install torchserve torch-model-archiver torch-workflow-archiver
     ```
 
 Now you are ready to [package and serve models with TorchServe](#serve-a-model).
@@ -71,7 +75,7 @@ Ensure that you have `python3` installed, and the user has access to the site-pa
 
 Run the following script from the top of the source directory.
 
-NOTE: This script uninstalls existing `torchserve` and `torch-model-archiver` installations
+NOTE: This script uninstalls existing `torchserve`, `torch-model-archiver` and `torch-workflow-archiver` installations
 
 #### For Debian Based Systems/ MacOS
 
@@ -136,7 +140,7 @@ torchserve --start --ncs --model-store model_store --models densenet161.mar
 
 After you execute the `torchserve` command above, TorchServe runs on your host, listening for inference requests.
 
-**Note**: If you specify model(s) when you run TorchServe, it automatically scales backend workers to the number equal to available vCPUs (if you run on a CPU instance) or to the number of available GPUs (if you run on a GPU instance). In case of powerful hosts with a lot of compute resoures (vCPUs or GPUs), this start up and autoscaling process might take considerable time. If you want to minimize TorchServe start up time you should avoid registering and scaling the model during start up time and move that to a later point by using corresponding [Management API](docs/management_api.md#register-a-model), which allows finer grain control of the resources that are allocated for any particular model).
+**Note**: If you specify model(s) when you run TorchServe, it automatically scales backend workers to the number equal to available vCPUs (if you run on a CPU instance) or to the number of available GPUs (if you run on a GPU instance). In case of powerful hosts with a lot of compute resources (vCPUs or GPUs), this start up and autoscaling process might take considerable time. If you want to minimize TorchServe start up time you should avoid registering and scaling the model during start up time and move that to a later point by using corresponding [Management API](docs/management_api.md#register-a-model), which allows finer grain control of the resources that are allocated for any particular model).
 
 ### Get predictions from a model
 
@@ -212,6 +216,11 @@ To stop the currently running TorchServe instance, run:
 torchserve --stop
 ```
 
+### Inspect the logs
+All the logs you've seen as output to stdout related to model registration, management, inference are recorded in the `/logs` folder.
+
+High level performance data like Throughput or Percentile Precision can be generated with [Benchmark](benchmark/README.md) and visualized in a report.
+
 ### Concurrency And Number of Workers
 TorchServe exposes configurations that allow the user to configure the number of worker threads on CPU and GPUs. There is an important config property that can speed up the server depending on the workload.
 *Note: the following property has bigger impact under heavy workloads.*
@@ -239,9 +248,12 @@ Feel free to skim the full list of [available examples](examples/README.md)
 ## Learn More
 
 * [Full documentation on TorchServe](docs/README.md)
-* [Manage models API](docs/management_api.md)
+* [Model Management API](docs/management_api.md)
 * [Inference API](docs/inference_api.md)
+* [Metrics API](docs/metrics.md)
 * [Package models for use with TorchServe](model-archiver/README.md)
+* [Deploying TorchServe with Kubernetes](kubernetes/README.md)
+* [TorchServe Workflows](examples/Workflows/README.md)
 * [TorchServe model zoo for pre-trained and pre-packaged models-archives](docs/model_zoo.md)
 
 ## Contributing
@@ -255,4 +267,4 @@ To file a bug or request a feature, please file a GitHub issue. For filing pull
 ## Disclaimer 
 This repository is jointly operated and maintained by Amazon, Facebook and a number of individual contributors listed in the [CONTRIBUTORS](https://github.com/pytorch/serve/graphs/contributors) file. For questions directed at Facebook, please send an email to [email protected]. For questions directed at Amazon, please send an email to [email protected]. For all other questions, please open up an issue in this repository [here](https://github.com/pytorch/serve/issues).
 
-*TorchServe acknowledges the [Multi Model Server (MMS)](https://github.com/awslabs/multi-model-server) project from which it was derived*
+*TorchServe acknowledges the [Multi Model Server (MMS)](https://github.com/awslabs/multi-model-server) project from which it was derived*

benchmarks/README.md

@@ -6,6 +6,7 @@ We currently support benchmarking with JMeter & Apache Bench. One can also profi
 
 * [Benchmarking with JMeter](#benchmarking-with-jmeter)
 * [Benchmarking with Apache Bench](#benchmarking-with-apache-bench)
+* [AutoBenchmarking Apachage Bench on AWS](#benchmarking-apache-bench-aws)
 * [Profiling](#profiling)
 
 # Benchmarking with JMeter
@@ -304,7 +305,7 @@ Note: These pre-defined parameters in test plan can be overwritten by cmd line a
 The reports are generated at location "/tmp/benchmark/"
 - CSV report: /tmp/benchmark/ab_report.csv
 - latency graph: /tmp/benchmark/predict_latency.png
-- torhcserve logs: /tmp/benchmark/logs/model_metrics.log
+- torchserve logs: /tmp/benchmark/logs/model_metrics.log
 - raw ab output: /tmp/benchmark/result.txt
 
 ### Sample output CSV
@@ -315,6 +316,9 @@ The reports are generated at location "/tmp/benchmark/"
 ### Sample latency graph
 ![](predict_latency.png)
 
+# Benchmarking Apache Bench AWS
+If you're making a large change to TorchServe it's best to run an [automated benchmarking suite on AWS](https://github.com/pytorch/serve/tree/master/test/benchmark) so that you can test multiple CUDA versions and EC2 hardware configurations easily.
+
 # Profiling
 
 ## Frontend