From c3e797cc10988d48a165cbac74d7903b9648b4cf Mon Sep 17 00:00:00 2001 From: VVsssssk Date: Mon, 21 Nov 2022 15:32:35 +0800 Subject: [PATCH 1/8] fix a bug --- .../layers/pointnet_modules/stack_point_sa_module.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py b/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py index d20ee510a1..e813a76a86 100644 --- a/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py +++ b/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py @@ -58,9 +58,10 @@ def forward(self, 'str(new_xyz_batch_cnt)' # idx: (M1 + M2 ..., nsample), empty_ball_mask: (M1 + M2 ...) - idx, empty_ball_mask = ball_query(0, self.radius, self.sample_nums, - xyz, new_xyz, xyz_batch_cnt, - new_xyz_batch_cnt) + idx = ball_query(0, self.radius, self.sample_nums, xyz, new_xyz, + xyz_batch_cnt, new_xyz_batch_cnt) + empty_ball_mask = (idx[:, 0] == -1) + idx[empty_ball_mask] = 0 grouped_xyz = grouping_operation( xyz, idx, xyz_batch_cnt, new_xyz_batch_cnt) # (M1 + M2, 3, nsample) From ecd709969cc796ddea61c71011c6e234669bf24d Mon Sep 17 00:00:00 2001 From: VVsssssk Date: Mon, 21 Nov 2022 16:35:02 +0800 Subject: [PATCH 2/8] fix a batch inference bug --- .../roi_heads/bbox_heads/pv_rcnn_bbox_head.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mmdet3d/models/roi_heads/bbox_heads/pv_rcnn_bbox_head.py b/mmdet3d/models/roi_heads/bbox_heads/pv_rcnn_bbox_head.py index 0cc9b6baee..61f21eecb7 100644 --- a/mmdet3d/models/roi_heads/bbox_heads/pv_rcnn_bbox_head.py +++ b/mmdet3d/models/roi_heads/bbox_heads/pv_rcnn_bbox_head.py @@ -4,6 +4,8 @@ import numpy as np import torch from mmcv.cnn import ConvModule +from mmdet.models.task_modules.samplers import SamplingResult +from mmdet.models.utils import multi_apply from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import nn as nn @@ -14,8 +16,6 @@ from mmdet3d.structures.bbox_3d import (LiDARInstance3DBoxes, rotation_3d_in_axis, xywhr2xyxyr) from mmdet3d.utils import InstanceList -from mmdet.models.task_modules.samplers import SamplingResult -from mmdet.models.utils import multi_apply @MODELS.register_module() @@ -440,21 +440,21 @@ def get_results(self, # post processing result_list = [] for batch_id in range(batch_size): - cls_preds = cls_preds[roi_batch_id == batch_id] + cur_cls_preds = cls_preds[roi_batch_id == batch_id] box_preds = batch_box_preds[roi_batch_id == batch_id] label_preds = class_labels[batch_id] - cls_preds = cls_preds.sigmoid() - cls_preds, _ = torch.max(cls_preds, dim=-1) + cur_cls_preds = cur_cls_preds.sigmoid() + cur_cls_preds, _ = torch.max(cur_cls_preds, dim=-1) selected = self.class_agnostic_nms( - scores=cls_preds, + scores=cur_cls_preds, bbox_preds=box_preds, input_meta=input_metas[batch_id], nms_cfg=test_cfg) selected_bboxes = box_preds[selected] selected_label_preds = label_preds[selected] - selected_scores = cls_preds[selected] + selected_scores = cur_cls_preds[selected] results = InstanceData() results.bboxes_3d = input_metas[batch_id]['box_type_3d']( From 05b96ed67e6b7fb87fbd0ac7bdb447f0925159da Mon Sep 17 00:00:00 2001 From: VVsssssk Date: Mon, 21 Nov 2022 17:06:13 +0800 Subject: [PATCH 3/8] fix docs --- mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py b/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py index e813a76a86..6283932781 100644 --- a/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py +++ b/mmdet3d/models/layers/pointnet_modules/stack_point_sa_module.py @@ -57,7 +57,7 @@ def forward(self, 'new_xyz: str(new_xyz.shape), new_xyz_batch_cnt: ' \ 'str(new_xyz_batch_cnt)' - # idx: (M1 + M2 ..., nsample), empty_ball_mask: (M1 + M2 ...) + # idx: (M1 + M2 ..., nsample) idx = ball_query(0, self.radius, self.sample_nums, xyz, new_xyz, xyz_batch_cnt, new_xyz_batch_cnt) empty_ball_mask = (idx[:, 0] == -1) From 578492a76e18b394dcb995aa0cd5f22eecabc4c2 Mon Sep 17 00:00:00 2001 From: VVsssssk Date: Tue, 22 Nov 2022 14:29:33 +0800 Subject: [PATCH 4/8] add pvrcnn benchmark --- configs/pv_rcnn/README.md | 42 +++++++++++++++++++ configs/pv_rcnn/metafile.yml | 29 +++++++++++++ .../pv_rcnn_8xb2-80e_kitti-3d-3class.py} | 0 .../test_models/test_detectors/test_pvrcnn.py | 2 +- 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 configs/pv_rcnn/README.md create mode 100644 configs/pv_rcnn/metafile.yml rename configs/{pvrcnn/pvrcnn_8xb2-80e_kitti-3d-3class.py => pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py} (100%) diff --git a/configs/pv_rcnn/README.md b/configs/pv_rcnn/README.md new file mode 100644 index 0000000000..4014cd2af2 --- /dev/null +++ b/configs/pv_rcnn/README.md @@ -0,0 +1,42 @@ +# PV-RCNN: Point-Voxel Feature Set Abstraction for 3D Object Detection + +> [PV-RCNN: Point-Voxel Feature Set Abstraction for 3D Object Detection](https://arxiv.org/abs/1912.13192) + + + +## Introduction + +3D object detection has been receiving increasing attention from both industry and academia thanks to its wide applications in various fields such as autonomous driving and robotics. LiDAR sensors are widely adopted in autonomous driving vehicles and robots for capturing 3D scene information as sparse and irregular point clouds, which provide vital cues for 3D scene perception and understanding. In this paper, we propose to achieve high performance 3D object detection by designing novel point-voxel integrated networks to learn better 3D features from irregular point clouds. + +
+ +
+ +## Results and models + +### KITTI + +| Backbone | Class | Lr schd | Mem (GB) | Inf time (fps) | mAP | Download | +| :---------------------------------------------: | :-----: | :--------: | :------: | :------------: | :---: | :----------------------: | +| [SECFPN](./pv_rcnn_8xb2-80e_kitti-3d-3class.py) | 3 Class | cyclic 80e | 5.4 | | 72.13 | [model](<>) \\ [log](<>) | + +Note: mAP represents AP11 results on 3 Class under the moderate setting. + +Detailed performance on KITTI 3D detection (3D) is as follows, evaluated by AP11 metric: + +| | Easy | Moderate | Hard | +| ---------- | :---: | :------: | :---: | +| Car | 89.26 | 83.46 | 78.78 | +| Pedestrian | 66.28 | 59.53 | 54.83 | +| Cyclist | 87.32 | 73.42 | 69.58 | + +## Citation + +```latex +@article{ShaoshuaiShi2020PVRCNNPF, + title={PV-RCNN: Point-Voxel Feature Set Abstraction for 3D Object Detection}, + author={Shaoshuai Shi and Chaoxu Guo and Li Jiang and Zhe Wang and Jianping Shi and Xiaogang Wang and Hongsheng Li}, + journal={computer vision and pattern recognition}, + year={2020} +} +``` diff --git a/configs/pv_rcnn/metafile.yml b/configs/pv_rcnn/metafile.yml new file mode 100644 index 0000000000..ef96a8d18f --- /dev/null +++ b/configs/pv_rcnn/metafile.yml @@ -0,0 +1,29 @@ +Collections: + - Name: PV-RCNN + Metadata: + Training Data: KITTI + Training Techniques: + - AdamW + Training Resources: 8x A100 GPUs + Architecture: + - Feature Pyramid Network + Paper: + URL: https://arxiv.org/abs/1912.13192 + Title: 'PV-RCNN: Point-Voxel Feature Set Abstraction for 3D Object Detection' + README: configs/pv_rcnn/README.md + Code: + URL: https://github.com/open-mmlab/mmdetection3d/blob/dev-1.x/mmdet3d/models/detectors/pv_rcnn.py#L12 + Version: v1.1.0rc2 + +Models: + - Name: pv_rcnn_8xb2-80e_kitti-3d-3class + In Collection: PV_RCNN + Config: configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py + Metadata: + Training Memory (GB): 5.4 + Results: + - Task: 3D Object Detection + Dataset: KITTI + Metrics: + mAP: 72.13 + Weights: diff --git a/configs/pvrcnn/pvrcnn_8xb2-80e_kitti-3d-3class.py b/configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py similarity index 100% rename from configs/pvrcnn/pvrcnn_8xb2-80e_kitti-3d-3class.py rename to configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py diff --git a/tests/test_models/test_detectors/test_pvrcnn.py b/tests/test_models/test_detectors/test_pvrcnn.py index 66efdeb074..5704e855bb 100644 --- a/tests/test_models/test_detectors/test_pvrcnn.py +++ b/tests/test_models/test_detectors/test_pvrcnn.py @@ -17,7 +17,7 @@ def test_pvrcnn(self): DefaultScope.get_instance('test_pvrcnn', scope_name='mmdet3d') _setup_seed(0) pvrcnn_cfg = _get_detector_cfg( - 'pvrcnn/pvrcnn_8xb2-80e_kitti-3d-3class.py') + 'pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py') model = MODELS.build(pvrcnn_cfg) num_gt_instance = 2 packed_inputs = _create_detector_inputs( From 1b759d42c9186dd8f3719f0a44db67532bddd90a Mon Sep 17 00:00:00 2001 From: VVsssssk Date: Tue, 22 Nov 2022 14:54:01 +0800 Subject: [PATCH 5/8] fix --- configs/pv_rcnn/README.md | 8 ++++---- configs/pv_rcnn/metafile.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/pv_rcnn/README.md b/configs/pv_rcnn/README.md index 4014cd2af2..29babeebf3 100644 --- a/configs/pv_rcnn/README.md +++ b/configs/pv_rcnn/README.md @@ -18,7 +18,7 @@ | Backbone | Class | Lr schd | Mem (GB) | Inf time (fps) | mAP | Download | | :---------------------------------------------: | :-----: | :--------: | :------: | :------------: | :---: | :----------------------: | -| [SECFPN](./pv_rcnn_8xb2-80e_kitti-3d-3class.py) | 3 Class | cyclic 80e | 5.4 | | 72.13 | [model](<>) \\ [log](<>) | +| [SECFPN](./pv_rcnn_8xb2-80e_kitti-3d-3class.py) | 3 Class | cyclic 80e | 5.4 | | 72.28 | [model](<>) \\ [log](<>) | Note: mAP represents AP11 results on 3 Class under the moderate setting. @@ -26,9 +26,9 @@ Detailed performance on KITTI 3D detection (3D) is as follows, evaluated by AP11 | | Easy | Moderate | Hard | | ---------- | :---: | :------: | :---: | -| Car | 89.26 | 83.46 | 78.78 | -| Pedestrian | 66.28 | 59.53 | 54.83 | -| Cyclist | 87.32 | 73.42 | 69.58 | +| Car | 89.20 | 83.72 | 78.79 | +| Pedestrian | 66.64 | 59.84 | 55.33 | +| Cyclist | 87.25 | 73.27 | 69.61 | ## Citation diff --git a/configs/pv_rcnn/metafile.yml b/configs/pv_rcnn/metafile.yml index ef96a8d18f..b452312f1a 100644 --- a/configs/pv_rcnn/metafile.yml +++ b/configs/pv_rcnn/metafile.yml @@ -25,5 +25,5 @@ Models: - Task: 3D Object Detection Dataset: KITTI Metrics: - mAP: 72.13 + mAP: 72.28 Weights: From 074264f62b01ba66d64e4aca3f245f6576a5a09a Mon Sep 17 00:00:00 2001 From: VVsssssk Date: Tue, 22 Nov 2022 17:07:08 +0800 Subject: [PATCH 6/8] add link --- configs/pv_rcnn/README.md | 6 +++--- configs/pv_rcnn/metafile.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/pv_rcnn/README.md b/configs/pv_rcnn/README.md index 29babeebf3..34527ca123 100644 --- a/configs/pv_rcnn/README.md +++ b/configs/pv_rcnn/README.md @@ -16,9 +16,9 @@ ### KITTI -| Backbone | Class | Lr schd | Mem (GB) | Inf time (fps) | mAP | Download | -| :---------------------------------------------: | :-----: | :--------: | :------: | :------------: | :---: | :----------------------: | -| [SECFPN](./pv_rcnn_8xb2-80e_kitti-3d-3class.py) | 3 Class | cyclic 80e | 5.4 | | 72.28 | [model](<>) \\ [log](<>) | +| Backbone | Class | Lr schd | Mem (GB) | Inf time (fps) | mAP | Download | +| :---------------------------------------------: | :-----: | :--------: | :------: | :------------: | :---: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | +| [SECFPN](./pv_rcnn_8xb2-80e_kitti-3d-3class.py) | 3 Class | cyclic 80e | 5.4 | | 72.28 | [model](https://download.openmmlab.com/mmdetection3d/v1.1.0_models/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class/pv_rcnn_8xb2-80e_kitti-3d-3class_20221117_234428-b384d22f.pth) \\ [log](https://download.openmmlab.com/mmdetection3d/v1.1.0_models/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class/pv_rcnn_8xb2-80e_kitti-3d-3class_20221117_234428.json) | Note: mAP represents AP11 results on 3 Class under the moderate setting. diff --git a/configs/pv_rcnn/metafile.yml b/configs/pv_rcnn/metafile.yml index b452312f1a..4e00805749 100644 --- a/configs/pv_rcnn/metafile.yml +++ b/configs/pv_rcnn/metafile.yml @@ -26,4 +26,4 @@ Models: Dataset: KITTI Metrics: mAP: 72.28 - Weights: + Weights: Date: Wed, 23 Nov 2022 14:54:11 +0800 Subject: [PATCH 7/8] add --- README.md | 2 ++ configs/pv_rcnn/metafile.yml | 2 +- docs/en/model_zoo.md | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8943e7ee40..32d8ed6cf8 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ Results and models are available in the [model zoo](docs/en/model_zoo.md).
  • PointRCNN (CVPR'2019)
  • Part-A2 (TPAMI'2020)
  • CenterPoint (CVPR'2021)
    • +
  • PV-RCNN (CVPR'2020)
  • Indoor
      • @@ -227,6 +228,7 @@ Results and models are available in the [model zoo](docs/en/model_zoo.md). | MonoFlex | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | ✗ | | SA-SSD | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ☐ | ☐ | ✗ | ✗ | | FCAF3D | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | +| PV-RCNN | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ☐ | ☐ | ✗ | ✗ | **Note:** All the about **300+ models, methods of 40+ papers** in 2D detection supported by [MMDetection](https://github.com/open-mmlab/mmdetection/blob/3.x/docs/en/model_zoo.md) can be trained or used in this codebase. diff --git a/configs/pv_rcnn/metafile.yml b/configs/pv_rcnn/metafile.yml index 4e00805749..efa6a8e342 100644 --- a/configs/pv_rcnn/metafile.yml +++ b/configs/pv_rcnn/metafile.yml @@ -17,7 +17,7 @@ Collections: Models: - Name: pv_rcnn_8xb2-80e_kitti-3d-3class - In Collection: PV_RCNN + In Collection: PV-RCNN Config: configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py Metadata: Training Memory (GB): 5.4 diff --git a/docs/en/model_zoo.md b/docs/en/model_zoo.md index 1a647ce980..e9bc20c3f2 100644 --- a/docs/en/model_zoo.md +++ b/docs/en/model_zoo.md @@ -104,6 +104,10 @@ Please refer to [MonoFlex](https://github.com/open-mmlab/mmdetection3d/tree/v1.0 Please refer to [SA-SSD](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/sassd) for details. We provide SA-SSD baselines on the KITTI dataset. +### PV-RCNN + +Please refer to [PV-RCNN](https://github.com/open-mmlab/mmdetection3d/blob/dev-1.x/configs/pv_rcnn) for details. We provide PV-RCNN baselines on the KITTI dataset. + ### Mixed Precision (FP16) Training Please refer to [Mixed Precision (FP16) Training on PointPillars](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0/configs/pointpillars/hv_pointpillars_fpn_sbn-all_fp16_2x8_2x_nus-3d.py) for details. From 5efeb080d0995dd439ac362b56cc711702cb7021 Mon Sep 17 00:00:00 2001 From: VVsssssk Date: Thu, 24 Nov 2022 11:04:18 +0800 Subject: [PATCH 8/8] fix lint --- .../roi_heads/mask_heads/foreground_segmentation_head.py | 2 +- mmdet3d/models/roi_heads/pv_rcnn_roi_head.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmdet3d/models/roi_heads/mask_heads/foreground_segmentation_head.py b/mmdet3d/models/roi_heads/mask_heads/foreground_segmentation_head.py index 65ea38fa04..c5581058d7 100644 --- a/mmdet3d/models/roi_heads/mask_heads/foreground_segmentation_head.py +++ b/mmdet3d/models/roi_heads/mask_heads/foreground_segmentation_head.py @@ -3,6 +3,7 @@ import torch from mmcv.cnn.bricks import build_norm_layer +from mmdet.models.utils import multi_apply from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import nn as nn @@ -10,7 +11,6 @@ from mmdet3d.models.builder import build_loss from mmdet3d.registry import MODELS from mmdet3d.utils import InstanceList -from mmdet.models.utils import multi_apply @MODELS.register_module() diff --git a/mmdet3d/models/roi_heads/pv_rcnn_roi_head.py b/mmdet3d/models/roi_heads/pv_rcnn_roi_head.py index adc001b0b1..1c6011195c 100644 --- a/mmdet3d/models/roi_heads/pv_rcnn_roi_head.py +++ b/mmdet3d/models/roi_heads/pv_rcnn_roi_head.py @@ -2,6 +2,8 @@ from typing import List, Optional import torch +from mmdet.models.task_modules import AssignResult +from mmdet.models.task_modules.samplers import SamplingResult from torch.nn import functional as F from mmdet3d.models.roi_heads.base_3droi_head import Base3DRoIHead @@ -9,8 +11,6 @@ from mmdet3d.structures import bbox3d2roi from mmdet3d.structures.det3d_data_sample import SampleList from mmdet3d.utils import InstanceList -from mmdet.models.task_modules import AssignResult -from mmdet.models.task_modules.samplers import SamplingResult @MODELS.register_module()