[Fix] fix torch allocator resouce releasing (#1708)

* delete root logger and add condition before calling caching_allocator_delete

* fix lint error

* use torch._C._cuda_cudaCachingAllocator_raw_delete

Author: AllentDan

mmdeploy/backend/tensorrt/torch_allocator.py

@@ -13,6 +13,7 @@ def __init__(self, device_id: int = 0) -> None:
 
         self.device_id = device_id
         self.mems = set()
+        self.caching_delete = torch._C._cuda_cudaCachingAllocator_raw_delete
 
     def __del__(self):
         """destructor."""
@@ -53,11 +54,9 @@ def deallocate(self: trt.IGpuAllocator, memory: int) -> bool:
         Returns:
             bool: deallocate success.
         """
-        logger = get_root_logger()
-        logger.debug(f'deallocate {memory} with TorchAllocator.')
         if memory not in self.mems:
             return False
 
-        torch.cuda.caching_allocator_delete(memory)
+        self.caching_delete(memory)
         self.mems.discard(memory)
         return True