From 2d319fa1523dd1cd9e058e590cc95cbc6c3a378f Mon Sep 17 00:00:00 2001 From: Evan Li Date: Thu, 30 May 2024 17:17:30 -0700 Subject: [PATCH 1/3] add Long dtype --- core/ir/Input.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/ir/Input.cpp b/core/ir/Input.cpp index 7fb5105a22..2296e7d7bc 100644 --- a/core/ir/Input.cpp +++ b/core/ir/Input.cpp @@ -15,6 +15,14 @@ bool valid_dtype_format_combo(nvinfer1::DataType dtype, nvinfer1::TensorFormat f default: return false; } + case nvinfer1::DataType::kINT64: // Supports just Linear (NCHW) + switch (format) { + case nvinfer1::TensorFormat::kLINEAR: + return true; + case nvinfer1::TensorFormat::kHWC: + default: + return false; + } case nvinfer1::DataType::kINT32: // Supports just Linear (NCHW) switch (format) { case nvinfer1::TensorFormat::kLINEAR: @@ -64,6 +72,8 @@ bool valid_input_dtype(nvinfer1::DataType dtype) { return true; case nvinfer1::DataType::kINT32: return true; + case nvinfer1::DataType::kINT64: + return true; default: return false; } From b17806e0949571161a5f0f504e1c0a39aeb4cb55 Mon Sep 17 00:00:00 2001 From: Evan Li Date: Mon, 17 Jun 2024 16:35:21 -0700 Subject: [PATCH 2/3] rebase and update to public TRT 10.1.0.27 --- README.md | 4 ++-- WORKSPACE | 14 ++++++-------- dev_dep_versions.yml | 2 +- docker/README.md | 4 ++-- examples/dynamo/requirements.txt | 2 +- packaging/pre_build_script_windows.sh | 2 +- packaging/smoke_test_script.sh | 2 +- py/ci/Dockerfile.ci | 12 ++++++------ py/requirements.txt | 2 +- pyproject.toml | 8 ++++---- tests/py/requirements.txt | 2 +- .../ci_workspaces/WORKSPACE.win.release.tmpl | 2 +- .../WORKSPACE.x86_64.release.rhel.tmpl | 4 ++-- 13 files changed, 29 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 9b436a2b0b..a39ccefd33 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Torch-TensorRT [![Documentation](https://img.shields.io/badge/docs-master-brightgreen)](https://nvidia.github.io/Torch-TensorRT/) [![pytorch](https://img.shields.io/badge/PyTorch-2.4-green)](https://www.python.org/downloads/release/python-31013/) [![cuda](https://img.shields.io/badge/CUDA-12.4-green)](https://developer.nvidia.com/cuda-downloads) -[![trt](https://img.shields.io/badge/TensorRT-10.0.1-green)](https://github.com/nvidia/tensorrt-llm) +[![trt](https://img.shields.io/badge/TensorRT-10.1.0-green)](https://github.com/nvidia/tensorrt-llm) [![license](https://img.shields.io/badge/license-BSD--3--Clause-blue)](./LICENSE) [![linux_tests](https://github.com/pytorch/TensorRT/actions/workflows/build-test-linux.yml/badge.svg)](https://github.com/pytorch/TensorRT/actions/workflows/build-test-linux.yml) [![windows_tests](https://github.com/pytorch/TensorRT/actions/workflows/build-test-windows.yml/badge.svg)](https://github.com/pytorch/TensorRT/actions/workflows/build-test-windows.yml) @@ -119,7 +119,7 @@ These are the following dependencies used to verify the testcases. Torch-TensorR - Bazel 6.3.2 - Libtorch 2.5.0.dev (latest nightly) (built with CUDA 12.4) - CUDA 12.4 -- TensorRT 10.0.1.6 +- TensorRT 10.1.0.27 ## Deprecation Policy diff --git a/WORKSPACE b/WORKSPACE index 734ce8c85f..654cb1c9b5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -84,25 +84,23 @@ http_archive( http_archive( name = "tensorrt", build_file = "@//third_party/tensorrt/archive:BUILD", - sha256 = "a5cd2863793d69187ce4c73b2fffc1f470ff28cfd91e3640017e53b8916453d5", - strip_prefix = "TensorRT-10.0.1.6", + sha256 = "606436ed219c72d1a25a889b2b0ae5cb5a68499dd6f944da4cabb3c34c067d55", + strip_prefix = "TensorRT-10.1.0.27", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.0.1/tars/TensorRT-10.0.1.6.Linux.x86_64-gnu.cuda-12.4.tar.gz", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.1.0/tars/TensorRT-10.1.0.27.Linux.x86_64-gnu.cuda-12.4.tar.gz", ], ) http_archive( name = "tensorrt_win", build_file = "@//third_party/tensorrt/archive:BUILD", - sha256 = "d667bd10b178e239b621a8929008ef3e27967d181bf07a39845a0f99edeec47a", - strip_prefix = "TensorRT-10.0.1.6", + sha256 = "2eb98008944945377eb328871a308704e95bf3bb295fc548784c6da41a70bbed", + strip_prefix = "TensorRT-10.1.0.27", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.0.1/zip/TensorRT-10.0.1.6.Windows10.win10.cuda-12.4.zip", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.1.0/zip/TensorRT-10.1.0.27.Windows.win10.cuda-12.4.zip", ], ) - - #################################################################################### # Locally installed dependencies (use in cases of custom dependencies or aarch64) #################################################################################### diff --git a/dev_dep_versions.yml b/dev_dep_versions.yml index 5c560c41f9..6272a30451 100644 --- a/dev_dep_versions.yml +++ b/dev_dep_versions.yml @@ -1,3 +1,3 @@ __version__: "2.5.0.dev0" __cuda_version__: "12.4" -__tensorrt_version__: "10.0.1" +__tensorrt_version__: "10.1.0" diff --git a/docker/README.md b/docker/README.md index 824dae4d54..a051a6ea9b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,14 +17,14 @@ Note: By default the container uses the `pre-cxx11-abi` version of Torch + Torch ### Instructions -- The example below uses TensorRT 10.0.1.6 +- The example below uses TensorRT 10.1.0.27 - See dependencies for a list of current default dependencies. > From root of Torch-TensorRT repo Build: ``` -DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.0.1 -f docker/Dockerfile -t torch_tensorrt:latest . +DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.1.0 -f docker/Dockerfile -t torch_tensorrt:latest . ``` Run: diff --git a/examples/dynamo/requirements.txt b/examples/dynamo/requirements.txt index a66d12bd1f..6cec09c35d 100644 --- a/examples/dynamo/requirements.txt +++ b/examples/dynamo/requirements.txt @@ -1,5 +1,5 @@ cupy==13.1.0 -tensorrt==10.0.1 +tensorrt==10.1.0 torch>=2.4.0.dev20240503+cu121 torch-tensorrt>=2.4.0.dev20240503+cu121 triton==2.3.0 diff --git a/packaging/pre_build_script_windows.sh b/packaging/pre_build_script_windows.sh index 9724e18327..b4893f81af 100644 --- a/packaging/pre_build_script_windows.sh +++ b/packaging/pre_build_script_windows.sh @@ -8,7 +8,7 @@ pip install tensorrt==${TRT_VERSION} tensorrt-${CU_VERSION::4}-bindings==${TRT_V choco install bazelisk -y -curl -Lo TensorRT.zip https://developer.download.nvidia.com/compute/machine-learning/tensorrt/10.0.1/zip/TensorRT-10.0.1.6.Windows10.win10.cuda-12.4.zip +curl -Lo TensorRT.zip https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.1.0/zip/TensorRT-10.1.0.27.Windows.win10.cuda-12.4.zip unzip -o TensorRT.zip -d C:/ export CUDA_HOME="$(echo ${CUDA_PATH} | sed -e 's#\\#\/#g')" diff --git a/packaging/smoke_test_script.sh b/packaging/smoke_test_script.sh index d3bed3249e..ea187eb5d1 100644 --- a/packaging/smoke_test_script.sh +++ b/packaging/smoke_test_script.sh @@ -2,5 +2,5 @@ # The issue was smoke test installs the built torch_tensorrt wheel file and checks `import torch_tensorrt; print(torch_tensorrt.__version__)` # Since tensorrt cannot be pip installable in CI, the smoke test will fail. # One way we tried to handle it is manually install tensorrt wheel while by extracting from the tarball. -# However, the TensorRT-10.0.0.6/lib path doesn't seem to show up in LD_LIBRARY_PATH even if we explicitly set it. +# However, the TensorRT-10.1.0.27/lib path doesn't seem to show up in LD_LIBRARY_PATH even if we explicitly set it. # TODO: Implement a custom smoke_test script to verify torch_tensorrt installation. \ No newline at end of file diff --git a/py/ci/Dockerfile.ci b/py/ci/Dockerfile.ci index 16495de77d..82a9dbdb7c 100644 --- a/py/ci/Dockerfile.ci +++ b/py/ci/Dockerfile.ci @@ -3,13 +3,13 @@ FROM pytorch/manylinux-builder:cuda12.4 RUN yum install -y ninja-build # download TensorRT tarball -RUN wget -q https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.0.1/tars/TensorRT-10.0.1.6.Linux.x86_64-gnu.cuda-12.4.tar.gz \ -&& gunzip TensorRT-10.0.1.6.Linux.x86_64-gnu.cuda-12.4.tar.gz \ -&& tar -xvf TensorRT-10.0.1.6.Linux.x86_64-gnu.cuda-12.4.tar \ -&& rm TensorRT-10.0.1.6.Linux.x86_64-gnu.cuda-12.4.tar +RUN wget -q https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.1.0/tars/TensorRT-10.1.0.27.Linux.x86_64-gnu.cuda-12.4.tar.gz \ +&& gunzip TensorRT-10.1.0.27.Linux.x86_64-gnu.cuda-12.4.tar.gz \ +&& tar -xvf TensorRT-10.1.0.27.Linux.x86_64-gnu.cuda-12.4.tar \ +&& rm TensorRT-10.1.0.27.Linux.x86_64-gnu.cuda-12.4.tar -ENV TENSORRT_DIR=/TensorRT-10.0.1.6 -ENV TENSORRT_VERSION=10.0.1 +ENV TENSORRT_DIR=/TensorRT-10.1.0.27 +ENV TENSORRT_VERSION=10.1.0 RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-amd64 \ && mv bazelisk-linux-amd64 /usr/bin/bazel \ diff --git a/py/requirements.txt b/py/requirements.txt index c68bdc4d4c..649ea09706 100644 --- a/py/requirements.txt +++ b/py/requirements.txt @@ -6,4 +6,4 @@ torch>=2.5.0.dev,<2.6.0 torchvision>=0.19.0.dev,<0.20.0 --extra-index-url https://pypi.ngc.nvidia.com pyyaml -tensorrt==10.0.1 +tensorrt==10.1.0 diff --git a/pyproject.toml b/pyproject.toml index f9e1f27690..5074bef153 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "cffi>=1.15.1", "typing-extensions>=4.7.0", "future>=0.18.3", - "tensorrt==10.0.1", + "tensorrt==10.1.0", "torch >=2.5.0.dev,<2.6.0", "pybind11==2.6.2", "numpy", @@ -42,9 +42,9 @@ requires-python = ">=3.8" keywords = ["pytorch", "torch", "tensorrt", "trt", "ai", "artificial intelligence", "ml", "machine learning", "dl", "deep learning", "compiler", "dynamo", "torchscript", "inference"] dependencies = [ "torch >=2.5.0.dev,<2.6.0", - "tensorrt==10.0.1", - "tensorrt-cu12_bindings==10.0.1", - "tensorrt-cu12_libs==10.0.1", + "tensorrt==10.1.0", + "tensorrt-cu12_bindings==10.1.0", + "tensorrt-cu12_libs==10.1.0", "packaging>=23", "numpy", "typing-extensions>=4.7.0", diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index bdae578713..de5bccd430 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -7,7 +7,7 @@ parameterized>=0.2.0 pytest>=8.2.1 pytest-xdist>=3.6.1 pyyaml -tensorrt==10.0.1 +tensorrt==10.1.0 timm>=1.0.3 transformers==4.39.3 --extra-index-url https://pypi.nvidia.com diff --git a/toolchains/ci_workspaces/WORKSPACE.win.release.tmpl b/toolchains/ci_workspaces/WORKSPACE.win.release.tmpl index b505efb944..ce3df15602 100644 --- a/toolchains/ci_workspaces/WORKSPACE.win.release.tmpl +++ b/toolchains/ci_workspaces/WORKSPACE.win.release.tmpl @@ -63,7 +63,7 @@ http_archive( new_local_repository( name = "tensorrt_win", - path = "C:/TensorRT-10.0.1.6", + path = "C:/TensorRT-10.1.0.27", build_file = "@//third_party/tensorrt/local:BUILD" ) diff --git a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl index 3dcde3cf56..69b2d62113 100644 --- a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl +++ b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl @@ -72,9 +72,9 @@ http_archive( name = "tensorrt", build_file = "@//third_party/tensorrt/archive:BUILD", sha256 = "a5cd2863793d69187ce4c73b2fffc1f470ff28cfd91e3640017e53b8916453d5", - strip_prefix = "TensorRT-10.0.1.6", + strip_prefix = "TensorRT-10.1.0.27", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.0.1/tars/TensorRT-10.0.1.6.Linux.x86_64-gnu.cuda-12.4.tar.gz", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.1.0/tars/TensorRT-10.1.0.27.Linux.x86_64-gnu.cuda-12.4.tar.gz", ], ) From 7a0c69dcc541e53f71dfbb3bb4cf65458603e02e Mon Sep 17 00:00:00 2001 From: Evan Li Date: Mon, 17 Jun 2024 18:18:43 -0700 Subject: [PATCH 3/3] fix sha256 --- toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl index 69b2d62113..3f900b5355 100644 --- a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl +++ b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl @@ -71,7 +71,7 @@ http_archive( http_archive( name = "tensorrt", build_file = "@//third_party/tensorrt/archive:BUILD", - sha256 = "a5cd2863793d69187ce4c73b2fffc1f470ff28cfd91e3640017e53b8916453d5", + sha256 = "606436ed219c72d1a25a889b2b0ae5cb5a68499dd6f944da4cabb3c34c067d55", strip_prefix = "TensorRT-10.1.0.27", urls = [ "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.1.0/tars/TensorRT-10.1.0.27.Linux.x86_64-gnu.cuda-12.4.tar.gz",