diff --git a/tools/perf/benchmark.sh b/tools/perf/benchmark.sh index ca53950b47..28094c7c3c 100644 --- a/tools/perf/benchmark.sh +++ b/tools/perf/benchmark.sh @@ -14,7 +14,7 @@ echo "Benchmarking VGG16 model" for bs in ${batch_sizes[@]} do python perf_run.py --model ${MODELS_DIR}/vgg16_scripted.jit.pt \ - --model_torch ${MODELS_DIR}/vgg16_pytorch.pt \ + --model_torch vgg16 \ --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \ --batch_size ${bs} \ --truncate \ @@ -27,7 +27,7 @@ echo "Benchmarking AlexNet model" for bs in ${batch_sizes[@]} do python perf_run.py --model ${MODELS_DIR}/alexnet_scripted.jit.pt \ - --model_torch ${MODELS_DIR}/alexnet_pytorch.pt \ + --model_torch alexnet \ --precision fp32,fp16 --inputs="(${bs}, 3, 227, 227)" \ --batch_size ${bs} \ --truncate \ @@ -40,7 +40,7 @@ echo "Benchmarking Resnet50 model" for bs in ${batch_sizes[@]} do python perf_run.py --model ${MODELS_DIR}/resnet50_scripted.jit.pt \ - --model_torch ${MODELS_DIR}/resnet50_pytorch.pt \ + --model_torch resnet50 \ --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \ --batch_size ${bs} \ --truncate \ @@ -53,7 +53,7 @@ echo "Benchmarking VIT model" for bs in ${batch_sizes[@]} do python perf_run.py --model ${MODELS_DIR}/vit_scripted.jit.pt \ - --model_torch ${MODELS_DIR}/vit_pytorch.pt \ + --model_torch vit \ --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \ --batch_size ${bs} \ --truncate \ @@ -66,19 +66,20 @@ echo "Benchmarking VIT Large model" for bs in ${large_model_batch_sizes[@]} do python perf_run.py --model ${MODELS_DIR}/vit_large_scripted.jit.pt \ - --model_torch ${MODELS_DIR}/vit_large_pytorch.pt \ + --model_torch vit_large \ --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \ --truncate \ --batch_size ${bs} \ --backends torch,ts_trt,dynamo,torch_compile,inductor \ --report "vit_large_perf_bs${bs}.txt" +done # Benchmark EfficientNet-B0 model echo "Benchmarking EfficientNet-B0 model" for bs in ${batch_sizes[@]} do python perf_run.py --model ${MODELS_DIR}/efficientnet_b0_scripted.jit.pt \ - --model_torch ${MODELS_DIR}/efficientnet_b0_pytorch.pt \ + --model_torch efficientnet_b0 \ --precision fp32,fp16 --inputs="(${bs}, 3, 224, 224)" \ --batch_size ${bs} \ --truncate \ @@ -90,7 +91,7 @@ done echo "Benchmarking SD UNet model" for bs in ${large_model_batch_sizes[@]} do - python perf_run.py --model_torch ${MODELS_DIR}/sd_unet_pytorch.pt \ + python perf_run.py --model_torch sd_unet \ --precision fp32,fp16 --inputs="(${bs}, 4, 128, 128)@fp16;(${bs})@fp16;(${bs}, 1, 768)@fp16" \ --batch_size ${bs} \ --backends torch,dynamo,torch_compile,inductor \ diff --git a/tools/perf/hub.py b/tools/perf/hub.py index 4a14b59a4b..22b685d1fc 100644 --- a/tools/perf/hub.py +++ b/tools/perf/hub.py @@ -67,9 +67,25 @@ def download_models(version_matches, manifest): # Download all models if torch version is different than model version if not version_matches: for n, m in BENCHMARK_MODELS.items(): + # Ensure all specified desired model formats exist and are valid + assert all( + (path in VALID_PATHS) + for path in ( + m["path"] if isinstance(m["path"], (list, tuple)) else [m["path"]] + ) + ), "Not all 'path' attributes in BENCHMARK_MODELS are valid" + manifest = get(n, m, manifest) else: for n, m in BENCHMARK_MODELS.items(): + # Ensure all specified desired model formats exist and are valid + assert all( + (path in VALID_PATHS) + for path in ( + m["path"] if isinstance(m["path"], (list, tuple)) else [m["path"]] + ) + ), "Not all 'path' attributes in BENCHMARK_MODELS are valid" + scripted_filename = "models/" + n + "_scripted.jit.pt" traced_filename = "models/" + n + "_traced.jit.pt" pytorch_filename = "models/" + n + "_pytorch.pt" @@ -147,12 +163,4 @@ def main(): if __name__ == "__main__": - # Ensure all specified desired model formats exist and are valid - paths = [ - [m["path"]] if isinstance(m["path"], str) else m["path"] - for m in BENCHMARK_MODELS.values() - ] - assert all( - (path in VALID_PATHS) for path_list in paths for path in path_list - ), "Not all 'path' attributes in BENCHMARK_MODELS are valid" main() diff --git a/tools/perf/models/.gitkeep b/tools/perf/models/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/perf/perf_run.py b/tools/perf/perf_run.py index 15ab892f57..d4d57c6d3f 100644 --- a/tools/perf/perf_run.py +++ b/tools/perf/perf_run.py @@ -15,7 +15,6 @@ # Importing supported Backends import torch import torch.backends.cudnn as cudnn -import torch_tensorrt as torchtrt from utils import ( BENCHMARK_MODELS, parse_backends, @@ -24,6 +23,8 @@ precision_to_dtype, ) +import torch_tensorrt as torchtrt + WARMUP_ITER = 10 results = [] @@ -465,6 +466,7 @@ def recordStats(backend, timings, precision, batch_size=1, compile_time_s=None): "Mean(FPS)": speed_mean, "Median-Latency(ms)": time_med * 1000, "Mean-Latency(ms)": time_mean * 1000, + "Latency-StdDev(ms)": time_std * 1000, "Compile Time(s)": compile_time_s, } results.append(stats) @@ -550,16 +552,6 @@ def recordStats(backend, timings, precision, batch_size=1, compile_time_s=None): model_torch = torch.load(model_name_torch).eval().cuda() elif model_name_torch in BENCHMARK_MODELS: model_torch = BENCHMARK_MODELS[model_name_torch]["model"].eval().cuda() - if model_name_torch == "bert_base_uncased": - from transformers.utils.fx import symbolic_trace as transformers_trace - - model_torch = ( - transformers_trace( - model_torch, input_names=["input_ids", "attention_mask"] - ) - .eval() - .cuda() - ) # If neither model type was provided if (model is None) and (model_torch is None): diff --git a/tools/perf/utils.py b/tools/perf/utils.py index dfa41a9829..41f49439a2 100644 --- a/tools/perf/utils.py +++ b/tools/perf/utils.py @@ -1,46 +1,83 @@ +from typing import Optional, Sequence, Union + import custom_models as cm import timm import torch -import torch_tensorrt import torchvision.models as models -BENCHMARK_MODELS = { - "vgg16": { - "model": models.vgg16(weights=models.VGG16_Weights.DEFAULT), - "path": ["script", "pytorch"], - }, - "alexnet": { - "model": models.alexnet(weights=models.AlexNet_Weights.DEFAULT), - "path": ["script", "pytorch"], - }, - "resnet50": { - "model": models.resnet50(weights=None), - "path": ["script", "pytorch"], - }, - "efficientnet_b0": { - "model": timm.create_model("efficientnet_b0", pretrained=True), - "path": ["script", "pytorch"], - }, - "vit": { - "model": timm.create_model("vit_base_patch16_224", pretrained=True), - "path": ["script", "pytorch"], - }, - "vit_large": { - "model": timm.create_model("vit_giant_patch14_224", pretrained=False), - "path": ["script", "pytorch"], - }, - "bert_base_uncased": { - "model": cm.BertModule(), - "inputs": cm.BertInputs(), - "path": ["trace", "pytorch"], - }, - "sd_unet": { - "model": cm.StableDiffusionUnet(), - "path": "pytorch", - }, +import torch_tensorrt + +BENCHMARK_MODEL_NAMES = { + "vgg16", + "alexnet", + "resnet50", + "efficientnet_b0", + "vit", + "vit_large", + "bert_base_uncased", + "sd_unet", } +class ModelStorage: + def __contains__(self, name: str): + return name in BENCHMARK_MODEL_NAMES + + def __getitem__(self, name: str): + assert name in BENCHMARK_MODEL_NAMES + + if name == "vgg16": + return { + "model": models.vgg16(weights=models.VGG16_Weights.DEFAULT), + "path": ["script", "pytorch"], + } + elif name == "alexnet": + return { + "model": models.alexnet(weights=models.AlexNet_Weights.DEFAULT), + "path": ["script", "pytorch"], + } + elif name == "resnet50": + return { + "model": models.resnet50(weights=None), + "path": ["script", "pytorch"], + } + elif name == "efficientnet_b0": + return { + "model": timm.create_model("efficientnet_b0", pretrained=True), + "path": ["script", "pytorch"], + } + elif name == "vit": + return { + "model": timm.create_model("vit_base_patch16_224", pretrained=True), + "path": ["script", "pytorch"], + } + elif name == "vit_large": + return { + "model": timm.create_model("vit_giant_patch14_224", pretrained=False), + "path": ["script", "pytorch"], + } + elif name == "bert_base_uncased": + return { + "model": cm.BertModule(), + "inputs": cm.BertInputs(), + "path": ["trace", "pytorch"], + } + elif name == "sd_unet": + return { + "model": cm.StableDiffusionUnet(), + "path": "pytorch", + } + else: + raise AssertionError(f"Invalid model name {name}") + + def items(self): + for name in BENCHMARK_MODEL_NAMES: + yield name, self.__getitem__(name) + + +BENCHMARK_MODELS = ModelStorage() + + def precision_to_dtype(pr): if pr == "fp32": return torch.float