Vacc Backend

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
       - id: flake8
         args: ["--exclude=*/client/inference_pb2.py,*/client/inference_pb2_grpc.py"]
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.11.5
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-yapf

configs/_base_/backends/vacc.py

@@ -0,0 +1,16 @@
+backend_config = dict(
+    type='vacc',
+    common_config=dict(name='end2end', model_info=None, vdsp_params_info=None),
+    model_inputs=[
+        dict(
+            shape=dict(input=[1, 3, 224, 224]),
+            qconfig=dict(
+                dtype='fp16',
+                calibrate_mode='percentile',
+                weight_scale='max',
+                data_transmode=1,
+                per_channel=False,
+                cluster_mode=0,
+                skip_conv_layers=[],
+            ))
+    ])

configs/mmcls/classification_vacc-fp16_static.py

@@ -0,0 +1,9 @@
+_base_ = ['./classification_static.py', '../_base_/backends/vacc.py']
+
+backend_config = dict(
+    common_config=dict(
+        model_info='/path/to/model_info.json',
+        vdsp_params_info='/path/to/vdsp_params_info.json'),
+    model_inputs=[
+        dict(shape=dict(input=[1, 3, 224, 224]), qconfig=dict(dtype='fp16'))
+    ])

configs/mmcls/classification_vacc-int8_static.py

@@ -0,0 +1,5 @@
+_base_ = ['./classification_static.py', '../_base_/backends/vacc.py']
+
+backend_config = dict(model_inputs=[
+    dict(shape=dict(input=[1, 3, 224, 224]), qconfig=dict(dtype='int8'))
+])

configs/mmdet/detection/detection_vacc-fp16_static_416x416.py

@@ -0,0 +1,20 @@
+_base_ = ['../_base_/base_static.py', '../../_base_/backends/vacc.py']
+
+backend_config = dict(
+    common_config=dict(
+        model_info='/path/to/model_info.json',
+        vdsp_params_info='/path/to/vdsp_params_info.json'),
+    model_inputs=[
+        dict(shape=dict(input=[1, 3, 416, 416]), qconfig=dict(dtype='fp16'))
+    ])
+
+partition_config = dict(
+    type='vacc_det',
+    apply_marks=True,
+    partition_cfg=[
+        dict(
+            save_file='model.onnx',
+            start=['detector_forward:input'],
+            end=['yolo_head:input'],
+            output_names=[f'pred_maps.{i}' for i in range(3)])
+    ])

configs/mmdet/detection/detection_vacc-int8_static_416x416.py

@@ -0,0 +1,16 @@
+_base_ = ['../_base_/base_static.py', '../../_base_/backends/vacc.py']
+
+backend_config = dict(model_inputs=[
+    dict(shape=dict(input=[1, 3, 416, 416]), qconfig=dict(dtype='int8'))
+])
+
+partition_config = dict(
+    type='vacc_det',
+    apply_marks=True,
+    partition_cfg=[
+        dict(
+            save_file='model.onnx',
+            start=['detector_forward:input'],
+            end=['yolo_head:input'],
+            output_names=[f'pred_maps.{i}' for i in range(3)])
+    ])

configs/mmseg/segmentation_vacc-fp16_static_512x512.py

@@ -0,0 +1,14 @@
+_base_ = ['./segmentation_static.py', '../_base_/backends/vacc.py']
+
+backend_config = dict(model_inputs=[dict(shape=dict(input=[1, 3, 512, 512]))])
+
+partition_config = dict(
+    type='end2end',
+    apply_marks=True,
+    partition_cfg=[
+        dict(
+            save_file='model.onnx',
+            start=['segmentor_forward:output'],
+            end=['seg_maps:input'],
+            output_names=['feat'])
+    ])

configs/mmseg/segmentation_vacc-int8_static_512x512.py

@@ -0,0 +1,16 @@
+_base_ = ['./segmentation_static.py', '../_base_/backends/vacc.py']
+
+backend_config = dict(model_inputs=[
+    dict(shape=dict(input=[1, 3, 512, 512]), qconfig=dict(dtype='int8'))
+])
+
+partition_config = dict(
+    type='end2end',
+    apply_marks=True,
+    partition_cfg=[
+        dict(
+            save_file='model.onnx',
+            start=['segmentor_forward:output'],
+            end=['decode_head:input'],
+            output_names=['feat'])
+    ])

docs/en/05-supported-backends/vacc.md

@@ -0,0 +1,92 @@
+# VACC Backend
+
+- cmake 3.10.0+
+- gcc/g++ 7.5.0
+- llvm 9.0.1
+- ubuntu 18.04
+
+## PCIE
+
+### 1.package
+
+- dkms (>=1.95)
+- linux-headers
+- dpkg (Ubuntu)
+- rpm  (CentOS)
+- python2
+- python3
+
+Check if there is a vacc card：`lspci -d:0100`
+
+1. Requirements
+
+   ```bash
+   sudo apt-get install dkms dpkg python2 python3
+   ```
+
+2. install driver
+
+   ```bash
+   sudo dpkg -i vastai-pci_xx.xx.xx.xx_xx.deb
+   ```
+
+3. Verify installation
+
+   ```bash
+   dpkg --status vastai-pci-xxx
+
+   #output
+   Package: vastai-pci-dkms
+   Status: install ok installed
+   ……
+   Version: xx.xx.xx.xx
+   Provides: vastai-pci-modules (= xx.xx.xx.xx)
+   Depends: dkms (>= 1.95)
+   Description: vastai-pci driver in DKMS format.
+
+   lsmod | grep vastai_pci
+
+   #output
+   vastai_pci        xxx  x
+   ```
+
+4. Upgrade driver
+
+   ```bash
+   sudo dpkg -i vastai-pci_dkms_xx.xx.xx.xx_xx.deb
+   ```
+
+5. Uninstall driver
+
+   ```bash
+   sudo dpkg -r vastai-pci_dkms_xx.xx.xx.xx_xx
+   ```
+
+### 2.reboot pcie
+
+```bash
+sudo chmod 666 /dev/kchar:0 && sudo echo reboot > /dev/kchar:0
+```
+
+## SDK
+
+### step.1
+
+```bash
+pip install -r requirements.txt
+```
+
+### step.2
+
+```bash
+sudo vi ~/.bashrc
+
+export VASTSTREAM_PIPELINE=true
+export VACC_IRTEXT_ENABLE=1
+export TVM_HOME="/opt/vastai/vaststream/tvm"
+export VASTSTREAM_HOME="/opt/vastai/vaststream/vacl"
+export LD_LIBRARY_PATH=$TVM_HOME/lib:$VASTSTREAM_HOME/lib
+export PYTHONPATH=$TVM_HOME/python:$TVM_HOME/vacc/python:$TVM_HOME/topi/python:${PYTHONPATH}:$VASTSTREAM_HOME/python
+
+source ~/.bashrc
+```

docs/zh_cn/05-supported-backends/vacc.md

@@ -0,0 +1,94 @@
+# VACC Backend
+
+- cmake 3.10.0+
+- gcc/g++ 7.5.0
+- llvm 9.0.1
+- ubuntu 18.04
+
+## PCIE
+
+### 1.package
+
+- dkms (>=1.95)
+- linux-headers
+- dpkg (Ubuntu)
+- rpm  (CentOS)
+- python2
+- python3
+
+查看是否有瀚博推理卡：`lspci -d:0100`
+
+1. 环境准备
+
+   ```bash
+   sudo apt-get install dkms dpkg python2 python3
+   ```
+
+2. driver安装
+
+   ```bash
+   sudo dpkg -i vastai-pci_xx.xx.xx.xx_xx.deb
+   ```
+
+3. 查看安装
+
+   ```bash
+   # 1.查看deb包是否安装成功
+   dpkg --status vastai-pci-xxx
+
+   #output
+   Package: vastai-pci-dkms
+   Status: install ok installed
+   ……
+   Version: xx.xx.xx.xx
+   Provides: vastai-pci-modules (= xx.xx.xx.xx)
+   Depends: dkms (>= 1.95)
+   Description: vastai-pci driver in DKMS format.
+
+   # 2.查看驱动是否已加载到内核
+   lsmod | grep vastai_pci
+
+   #output
+   vastai_pci        xxx  x
+   ```
+
+4. 升级驱动
+
+   ```bash
+   sudo dpkg -i vastai-pci_dkms_xx.xx.xx.xx_xx.deb
+   ```
+
+5. 卸载驱动
+
+   ```bash
+   sudo dpkg -r vastai-pci_dkms_xx.xx.xx.xx_xx
+   ```
+
+### 2.reboot pcie
+
+```bash
+sudo chmod 666 /dev/kchar:0 && sudo echo reboot > /dev/kchar:0
+```
+
+## SDK
+
+### step.1
+
+```bash
+pip install -r requirements.txt
+```
+
+### step.2
+
+```bash
+sudo vi ~/.bashrc
+
+export VASTSTREAM_PIPELINE=true
+export VACC_IRTEXT_ENABLE=1
+export TVM_HOME="/opt/vastai/vaststream/tvm"
+export VASTSTREAM_HOME="/opt/vastai/vaststream/vacl"
+export LD_LIBRARY_PATH=$TVM_HOME/lib:$VASTSTREAM_HOME/lib
+export PYTHONPATH=$TVM_HOME/python:$TVM_HOME/vacc/python:$TVM_HOME/topi/python:${PYTHONPATH}:$VASTSTREAM_HOME/python
+
+source ~/.bashrc
+```

mmdeploy/apis/vacc/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from mmdeploy.backend.vacc import is_available
+from ..core import PIPELINE_MANAGER
+
+__all__ = ['is_available']
+
+if is_available():
+    try:
+        from mmdeploy.backend.vacc import from_onnx as _from_onnx
+        from_onnx = PIPELINE_MANAGER.register_pipeline()(_from_onnx)
+        __all__ += ['from_onnx']
+    except Exception:
+        pass

mmdeploy/backend/vacc/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .backend_manager import VACCManager
+from .onnx2vacc import from_onnx
+
+_BackendManager = VACCManager
+
+is_available = _BackendManager.is_available
+build_wrapper = _BackendManager.build_wrapper
+
+__all__ = ['VACCManager', 'from_onnx']
+
+if is_available():
+    try:
+        from .wrapper import VACCWrapper
+
+        __all__ += ['VACCWrapper']
+    except Exception:
+        pass