[Fix] Fix unit test in dev1.x

docs/en/01-how-to-build/rockchip.md

@@ -140,7 +140,7 @@ label: 65, score: 0.95
 
 - MMDet models.
 
-  YOLOV3 & YOLOX: you may paste the following partition configuration into [detection_rknn_static-320x320.py](https://github.com/open-mmlab/mmdeploy/blob/1.x/configs/mmdet/detection/detection_rknn_static-320x320.py):
+  YOLOV3 & YOLOX: you may paste the following partition configuration into [detection_rknn_static-320x320.py](https://github.com/open-mmlab/mmdeploy/blob/1.x/configs/mmdet/detection/detection_rknn-int8_static-320x320.py):
 
   ```python
   # yolov3, yolox for rknn-toolkit and rknn-toolkit2
@@ -172,7 +172,7 @@ label: 65, score: 0.95
       ])
   ```
 
-  RetinaNet & SSD & FSAF with rknn-toolkit2, you may paste the following partition configuration into [detection_rknn_static-320x320.py](https://github.com/open-mmlab/mmdeploy/tree/1.x/configs/mmdet/detection/detection_rknn_static-320x320.py). Users with rknn-toolkit can directly use default config.
+  RetinaNet & SSD & FSAF with rknn-toolkit2, you may paste the following partition configuration into [detection_rknn_static-320x320.py](https://github.com/open-mmlab/mmdeploy/blob/dev-1.x/configs/mmdet/detection/detection_rknn-int8_static-320x320.py). Users with rknn-toolkit can directly use default config.
 
   ```python
   # retinanet, ssd for rknn-toolkit2

docs/zh_cn/01-how-to-build/rockchip.md

@@ -105,7 +105,7 @@ python tools/deploy.py \
 
 - YOLOV3 & YOLOX
 
-将下面的模型拆分配置写入到 [detection_rknn_static.py](https://github.com/open-mmlab/mmdeploy/blob/1.x/configs/mmdet/detection/detection_rknn_static-320x320.py)
+将下面的模型拆分配置写入到 [detection_rknn_static.py](https://github.com/open-mmlab/mmdeploy/blob/1.x/configs/mmdet/detection/detection_rknn-int8_static-320x320.py)
 
 ```python
 # yolov3, yolox for rknn-toolkit and rknn-toolkit2
@@ -154,7 +154,7 @@ partition_config = dict(
 
 - RetinaNet & SSD & FSAF with rknn-toolkit2
 
-将下面的模型拆分配置写入到 [detection_rknn_static.py](https://github.com/open-mmlab/mmdeploy/blob/1.x/configs/mmdet/detection/detection_rknn_static-320x320.py)。使用 rknn-toolkit 的用户则不用。
+将下面的模型拆分配置写入到 [detection_rknn_static.py](https://github.com/open-mmlab/mmdeploy/blob/1.x/configs/mmdet/detection/detection_rknn-int8_static-320x320.py)。使用 rknn-toolkit 的用户则不用。
 
 ```python
 # retinanet, ssd and fsaf for rknn-toolkit2

mmdeploy/codebase/mmdet/models/dense_heads/detr_head.py

@@ -46,7 +46,7 @@ def detrhead__predict_by_feat__default(self,
     bbox_preds = all_bbox_preds_list[-1][-1]
 
     img_shape = batch_img_metas[0]['img_shape']
-    max_per_img = self.test_cfg.get('max_per_img', self.num_query)
+    max_per_img = self.test_cfg.get('max_per_img', len(cls_scores[0]))
     batch_size = cls_scores.size(0)
     # `batch_index_offset` is used for the gather of concatenated tensor
 

mmdeploy/codebase/mmedit/deploy/super_resolution_model.py

@@ -3,7 +3,7 @@
 
 import mmengine
 import torch
-from mmedit.structures import EditDataSample, PixelData
+from mmedit.structures import EditDataSample
 from mmengine import Config
 from mmengine.model.base_model.data_preprocessor import BaseDataPreprocessor
 from mmengine.registry import Registry
@@ -64,6 +64,35 @@ def _init_wrapper(self, backend: Backend, backend_files: Sequence[str],
             deploy_cfg=self.deploy_cfg,
             **kwargs)
 
+    def convert_to_datasample(
+            self, predictions: EditDataSample, data_samples: EditDataSample,
+            inputs: Optional[torch.Tensor]) -> List[EditDataSample]:
+        """Add predictions and destructed inputs (if passed) to data samples.
+
+        Args:
+            predictions (EditDataSample): The predictions of the model.
+            data_samples (EditDataSample): The data samples loaded from
+                dataloader.
+            inputs (Optional[torch.Tensor]): The input of model. Defaults to
+                None.
+
+        Returns:
+            List[EditDataSample]: Modified data samples.
+        """
+
+        if inputs is not None:
+            destructed_input = self.data_preprocessor.destruct(
+                inputs, data_samples, 'img')
+            data_samples.set_tensor_data({'input': destructed_input})
+        # split to list of data samples
+        data_samples = data_samples.split()
+        predictions = predictions.split()
+
+        for data_sample, pred in zip(data_samples, predictions):
+            data_sample.output = pred
+
+        return data_samples
+
     def forward(self,
                 inputs: torch.Tensor,
                 data_samples: Optional[List[BaseDataElement]] = None,
@@ -94,20 +123,15 @@ def forward(self,
         lq = lq.to(self.device)
         batch_outputs = self.wrapper({self.input_name:
                                       lq})[self.output_names[0]].to('cpu')
-        if hasattr(self.data_preprocessor, 'destructor'):
-            batch_outputs = self.data_preprocessor.destructor(
-                batch_outputs.to(self.data_preprocessor.outputs_std.device))
-        predictions = []
-
-        for sr_pred, data_sample in zip(batch_outputs, data_samples):
-            pred = EditDataSample()
-            pred.set_data(dict(pred_img=PixelData(**dict(data=sr_pred))))
-            data_sample.set_data(dict(output=pred))
-            '''
-            data_sample.set_data(
-                dict(pred_img=PixelData(**dict(data=sr_pred))))
-            '''
-            predictions.append(data_sample)
+        assert hasattr(self.data_preprocessor, 'destruct')
+        batch_outputs = self.data_preprocessor.destruct(
+            batch_outputs, data_samples)
+
+        # create a stacked data sample here
+        predictions = EditDataSample(pred_img=batch_outputs.cpu())
+
+        predictions = self.convert_to_datasample(predictions, data_samples,
+                                                 inputs)
         return predictions
 
 
@@ -118,6 +142,35 @@ class SDKEnd2EndModel(End2EndModel):
     def __init__(self, *args, **kwargs):
         super(SDKEnd2EndModel, self).__init__(*args, **kwargs)
 
+    def convert_to_datasample(
+            self, predictions: EditDataSample, data_samples: EditDataSample,
+            inputs: Optional[torch.Tensor]) -> List[EditDataSample]:
+        """Add predictions and destructed inputs (if passed) to data samples.
+
+        Args:
+            predictions (EditDataSample): The predictions of the model.
+            data_samples (EditDataSample): The data samples loaded from
+                dataloader.
+            inputs (Optional[torch.Tensor]): The input of model. Defaults to
+                None.
+
+        Returns:
+            List[EditDataSample]: Modified data samples.
+        """
+
+        if inputs is not None:
+            destructed_input = self.data_preprocessor.destruct(
+                inputs, data_samples, 'img')
+            data_samples.set_tensor_data({'input': destructed_input})
+        # split to list of data samples
+        data_samples = data_samples.split()
+        predictions = predictions.split()
+
+        for data_sample, pred in zip(data_samples, predictions):
+            data_sample.output = pred
+
+        return data_samples
+
     def forward(self,
                 inputs: torch.Tensor,
                 data_samples: Optional[List[BaseDataElement]] = None,
@@ -151,14 +204,14 @@ def forward(self,
             outputs.append(
                 torch.from_numpy(output).permute(2, 0, 1).contiguous())
         outputs = torch.stack(outputs, 0) / 255.
-        if hasattr(self.data_preprocessor, 'destructor'):
-            outputs = self.data_preprocessor.destructor(
-                outputs.to(self.data_preprocessor.outputs_std.device))
+        assert hasattr(self.data_preprocessor, 'destruct')
+        outputs = self.data_preprocessor.destruct(outputs, data_samples)
+
+        # create a stacked data sample here
+        predictions = EditDataSample(pred_img=outputs.cpu())
 
-        for i, sr_pred in enumerate(outputs):
-            pred = EditDataSample()
-            pred.set_data(dict(pred_img=PixelData(**dict(data=sr_pred))))
-            data_samples[i].set_data(dict(output=pred))
+        predictions = self.convert_to_datasample(predictions, data_samples,
+                                                 inputs)
         return data_samples
 
 

mmdeploy/mmcv/ops/nms_rotated.py

@@ -26,8 +26,7 @@ def forward(ctx, boxes: Tensor, scores: Tensor, iou_threshold: float,
         Returns:
             Tensor: Selected indices of boxes.
         """
-        from mmcv.utils import ext_loader
-        ext_module = ext_loader.load_ext('_ext', ['nms_rotated'])
+        from mmcv.ops import nms_rotated
         batch_size, num_class, _ = scores.shape
 
         indices = []
@@ -42,11 +41,8 @@ def forward(ctx, boxes: Tensor, scores: Tensor, iou_threshold: float,
                     continue
                 valid_inds = torch.nonzero(
                     valid_mask, as_tuple=False).squeeze(dim=1)
-                _, order = _scores.sort(0, descending=True)
-                dets_sorted = _boxes.index_select(0, order)
-                box_inds = ext_module.nms_rotated(_boxes, _scores, order,
-                                                  dets_sorted, iou_threshold,
-                                                  0)
+                _, box_inds = nms_rotated(
+                    _boxes, _scores, iou_threshold=iou_threshold)
                 box_inds = valid_inds[box_inds]
                 batch_inds = torch.zeros_like(box_inds) + batch_id
                 cls_inds = torch.zeros_like(box_inds) + cls_id

tests/test_apis/test_torch2torchscript.py

@@ -42,12 +42,11 @@ def test_torch2torchscript(input_name, output_name):
     import numpy as np
     deploy_cfg = get_deploy_cfg(input_name, output_name)
     torch2torchscript(
-        np.random.rand(8, 8, 3),
+        np.random.randint(0, 255, (8, 8, 3)),
         '',
         ts_file,
         deploy_cfg,
         model_cfg=get_model_cfg(),
         device='cpu')
 
-    print(ts_file)
     assert osp.exists(ts_file)

tests/test_codebase/test_mmaction/test_mmaction_model.py

@@ -29,7 +29,7 @@ def test_forward_of_base_recognizer(model_cfg_path, backend):
                 keep_initializers_as_inputs=False,
                 opset_version=11,
                 input_shape=None,
-                input_names=['input'],
+                input_names=['inputs'],
                 output_names=['output'])))
 
     model_cfg = load_config(model_cfg_path)[0]

tests/test_codebase/test_mmcls/test_mmcls_models.py

@@ -29,7 +29,7 @@ def get_invertedresidual_model():
     return model
 
 
-def get_vit_model():
+def get_vit_backbone():
     from mmcls.models.classifiers.image import ImageClassifier
     model = ImageClassifier(
         backbone={
@@ -60,7 +60,7 @@ def get_vit_model():
             },
             'topk': (1, 5)
         },
-    )
+    ).backbone
     model.requires_grad_(False)
 
     return model
@@ -151,39 +151,30 @@ def test_shufflenetv2_backbone__forward(backend_type: Backend):
 
 @pytest.mark.parametrize('backend_type', [Backend.NCNN])
 def test_vision_transformer_backbone__forward(backend_type: Backend):
-
-    from mmcls.structures import ClsDataSample
-
-    from mmdeploy.core import patch_model
     import_codebase(Codebase.MMCLS)
     check_backend(backend_type, True)
-    model = get_vit_model()
+    model = get_vit_backbone()
     model.eval()
 
     deploy_cfg = Config(
         dict(
             backend_config=dict(type=backend_type.value),
-            onnx_config=dict(input_shape=None, output_names=['output']),
+            onnx_config=dict(input_shape=None, output_names=['out0', 'out1']),
             codebase_config=dict(type='mmcls', task='Classification')))
 
     imgs = torch.rand((1, 3, 384, 384))
-    data_sample = ClsDataSample(
-        metainfo=dict(
-            scale_factor=(1, 1),
-            ori_shape=imgs.shape[2:],
-            img_shape=imgs.shape[2:]))
-    model = patch_model(
-        model, {}, backend=backend_type.value, data_samples=[data_sample])
-    model_outputs = model.forward(imgs)
+    model_outputs = model.forward(imgs)[0]
     wrapped_model = WrapModel(model, 'forward')
-    rewrite_inputs = {'batch_inputs': imgs}
+    rewrite_inputs = {'x': imgs}
     rewrite_outputs, is_backend_output = get_rewrite_outputs(
         wrapped_model=wrapped_model,
         model_inputs=rewrite_inputs,
         deploy_cfg=deploy_cfg)
 
     if isinstance(rewrite_outputs, dict):
-        rewrite_outputs = rewrite_outputs['output']
+        rewrite_outputs = [
+            rewrite_outputs[out_name] for out_name in ['out0', 'out1']
+        ]
     for model_output, rewrite_output in zip(model_outputs, rewrite_outputs):
         if isinstance(rewrite_output, torch.Tensor):
             rewrite_output = rewrite_output.cpu().numpy()

tests/test_codebase/test_mmdet/test_mmdet_models.py

@@ -406,54 +406,15 @@ def get_detrhead_model():
         dict(
             type='DETRHead',
             num_classes=4,
-            in_channels=1,
-            transformer=dict(
-                type='Transformer',
-                encoder=dict(
-                    type='DetrTransformerEncoder',
-                    num_layers=1,
-                    transformerlayers=dict(
-                        type='BaseTransformerLayer',
-                        attn_cfgs=[
-                            dict(
-                                type='MultiheadAttention',
-                                embed_dims=4,
-                                num_heads=1)
-                        ],
-                        ffn_cfgs=dict(
-                            type='FFN',
-                            embed_dims=4,
-                            feedforward_channels=32,
-                            num_fcs=2,
-                            ffn_drop=0.,
-                            act_cfg=dict(type='ReLU', inplace=True),
-                        ),
-                        operation_order=('self_attn', 'norm', 'ffn', 'norm'))),
-                decoder=dict(
-                    type='DetrTransformerDecoder',
-                    return_intermediate=True,
-                    num_layers=1,
-                    transformerlayers=dict(
-                        type='DetrTransformerDecoderLayer',
-                        attn_cfgs=dict(
-                            type='MultiheadAttention',
-                            embed_dims=4,
-                            num_heads=1),
-                        ffn_cfgs=dict(
-                            type='FFN',
-                            embed_dims=4,
-                            feedforward_channels=32,
-                            num_fcs=2,
-                            ffn_drop=0.,
-                            act_cfg=dict(type='ReLU', inplace=True),
-                        ),
-                        feedforward_channels=32,
-                        operation_order=('self_attn', 'norm', 'cross_attn',
-                                         'norm', 'ffn', 'norm')),
-                )),
-            positional_encoding=dict(
-                type='SinePositionalEncoding', num_feats=2, normalize=True),
-            test_cfg=dict(max_per_img=100)))
+            embed_dims=4,
+            loss_cls=dict(
+                type='CrossEntropyLoss',
+                bg_cls_weight=0.1,
+                use_sigmoid=False,
+                loss_weight=1.0,
+                class_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=5.0),
+            loss_iou=dict(type='GIoULoss', loss_weight=2.0)))
     model.requires_grad_(False)
     return model
 
@@ -715,7 +676,7 @@ def test_forward_of_base_detector(model_cfg_path, backend):
     model_cfg = Config(dict(model=mmengine.load(model_cfg_path)))
     model_cfg.model = _replace_r50_with_r18(model_cfg.model)
     from mmdet.apis import init_detector
-    model = init_detector(model_cfg, None, device='cpu')
+    model = init_detector(model_cfg, None, device='cpu', palette='coco')
 
     img = torch.randn(1, 3, 64, 64)
     from mmdet.structures import DetDataSample