diff --git a/modules/alphamat/CMakeLists.txt b/modules/alphamat/CMakeLists.txt index f5c9d8917f2..32fca08a15d 100644 --- a/modules/alphamat/CMakeLists.txt +++ b/modules/alphamat/CMakeLists.txt @@ -6,4 +6,5 @@ endif() ocv_define_module(alphamat opencv_core opencv_imgproc + WRAP python ) diff --git a/modules/alphamat/README.md b/modules/alphamat/README.md index e3dbe6bf443..abddf9b601f 100644 --- a/modules/alphamat/README.md +++ b/modules/alphamat/README.md @@ -7,12 +7,12 @@ This project was part of the Google Summer of Code 2019. *** Alphamatting is the problem of extracting the foreground from an image. Given the input of an image and its corresponding trimap, we try to extract the foreground from the background. -This project is implementation of "[[Designing Effective Inter-Pixel Information Flow for Natural Image Matting](http://people.inf.ethz.ch/aksoyy/ifm/)]" by Yağız Aksoy, Tunç Ozan Aydın and Marc Pollefeys[1]. It required implementation of parts of other papers [2,3,4]. +This project is implementation of "[Designing Effective Inter-Pixel Information Flow for Natural Image Matting](https://www.researchgate.net/publication/318489370_Designing_Effective_Inter-Pixel_Information_Flow_for_Natural_Image_Matting)" by Yağız Aksoy, Tunç Ozan Aydın and Marc Pollefeys[1]. It required implementation of parts of other papers [2,3,4]. ## References -[1] Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "[Designing Effective Inter-Pixel Information Flow for Natural Image Matting](http://people.inf.ethz.ch/aksoyy/ifm/)", CVPR, 2017. +[1] Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "[Designing Effective Inter-Pixel Information Flow for Natural Image Matting](https://www.researchgate.net/publication/318489370_Designing_Effective_Inter-Pixel_Information_Flow_for_Natural_Image_Matting)", CVPR, 2017. [2] Roweis, Sam T., and Lawrence K. Saul. "[Nonlinear dimensionality reduction by locally linear embedding](https://science.sciencemag.org/content/290/5500/2323)" Science 290.5500 (2000): 2323-2326. diff --git a/modules/alphamat/include/opencv2/alphamat.hpp b/modules/alphamat/include/opencv2/alphamat.hpp index 927fbb09eac..003b305e5e6 100644 --- a/modules/alphamat/include/opencv2/alphamat.hpp +++ b/modules/alphamat/include/opencv2/alphamat.hpp @@ -9,8 +9,15 @@ /** * @defgroup alphamat Alpha Matting - * This module is dedicated to compute alpha matting of images, given the input image and an input trimap. - * The samples directory includes easy examples of how to use the module. + * Alpha matting is used to extract a foreground object with soft boundaries from a background image. + * + * This module is dedicated to computing alpha matte of objects in images from a given input image and a greyscale trimap image that contains information about the foreground, background and unknown pixels. The unknown pixels are assumed to be a combination of foreground and background pixels. The algorithm uses a combination of multiple carefully defined pixels affinities to estimate the opacity of the foreground pixels in the unkown region. + * + * The implementation is based on @cite aksoy2017designing. + * + * This module was developed by Muskaan Kularia and Sunita Nayak as a project + * for Google Summer of Code 2019 (GSoC 19). + * */ namespace cv { namespace alphamat { @@ -18,10 +25,12 @@ namespace cv { namespace alphamat { //! @{ /** - * The implementation is based on Designing Effective Inter-Pixel Information Flow for Natural Image Matting by Yağız Aksoy, Tunç Ozan Aydın and Marc Pollefeys, CVPR 2019. + * @brief Compute alpha matte of an object in an image + * @param image Input RGB image + * @param tmap Input greyscale trimap image + * @param result Output alpha matte image * - * This module has been originally developed by Muskaan Kularia and Sunita Nayak as a project - * for Google Summer of Code 2019 (GSoC 19). + * The function infoFlow performs alpha matting on a RGB image using a greyscale trimap image, and outputs a greyscale alpha matte image. The output alpha matte can be used to softly extract the foreground object from a background image. Examples can be found in the samples directory. * */ CV_EXPORTS_W void infoFlow(InputArray image, InputArray tmap, OutputArray result); diff --git a/modules/alphamat/samples/information_flow_matting.cpp b/modules/alphamat/samples/information_flow_matting.cpp index f4dbda1d002..679111ea03d 100644 --- a/modules/alphamat/samples/information_flow_matting.cpp +++ b/modules/alphamat/samples/information_flow_matting.cpp @@ -2,16 +2,19 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +// Include relevant headers #include #include "opencv2/highgui.hpp" #include #include #include +// Set namespaces using namespace std; using namespace cv; using namespace cv::alphamat; +// Set the usage parameter names const char* keys = "{img || input image name}" "{tri || input trimap image name}" @@ -30,10 +33,12 @@ int main(int argc, char* argv[]) return 0; } + // Read the paths to the input image, input trimap and the location of the output image. string img_path = parser.get("img"); string trimap_path = parser.get("tri"); string result_path = parser.get("out"); + // Make sure the user inputs paths to the input image and trimap if (!parser.check() || img_path.empty() || trimap_path.empty()) { @@ -44,13 +49,15 @@ int main(int argc, char* argv[]) Mat image, tmap; - image = imread(img_path, IMREAD_COLOR); // Read the input image file + // Read the input image + image = imread(img_path, IMREAD_COLOR); if (image.empty()) { printf("Cannot read image file: '%s'\n", img_path.c_str()); return 1; } + // Read the trimap tmap = imread(trimap_path, IMREAD_GRAYSCALE); if (tmap.empty()) { @@ -59,16 +66,19 @@ int main(int argc, char* argv[]) } Mat result; + // Perform information flow alpha matting infoFlow(image, tmap, result); if (result_path.empty()) { + // Show the alpha matte if a result filepath is not provided. namedWindow("result alpha matte", WINDOW_NORMAL); imshow("result alpha matte", result); waitKey(0); } else { + // Save the alphamatte imwrite(result_path, result); printf("Result saved: '%s'\n", result_path.c_str()); } diff --git a/modules/alphamat/samples/output_mattes/plant_result.jpg b/modules/alphamat/samples/output_mattes/plant_result.jpg deleted file mode 100644 index 4ec7e29c6b0..00000000000 Binary files a/modules/alphamat/samples/output_mattes/plant_result.jpg and /dev/null differ diff --git a/modules/alphamat/samples/output_mattes/plant_result.png b/modules/alphamat/samples/output_mattes/plant_result.png new file mode 100644 index 00000000000..f9a2e93f1dd Binary files /dev/null and b/modules/alphamat/samples/output_mattes/plant_result.png differ diff --git a/modules/alphamat/tutorials/alphamat_tutorial.markdown b/modules/alphamat/tutorials/alphamat_tutorial.markdown index 03cd329f5f2..db9f6f72fad 100644 --- a/modules/alphamat/tutorials/alphamat_tutorial.markdown +++ b/modules/alphamat/tutorials/alphamat_tutorial.markdown @@ -7,15 +7,26 @@ This project was part of Google Summer of Code 2019. *Mentor:* Sunita Nayak -Alphamatting is the problem of extracting the foreground from an image. The extracted foreground can be used for further operations like changing the background in an image. +Alphamatting is the problem of extracting the foreground with soft boundaries from a background image. The extracted foreground can be used for further operations like changing the background in an image. Given an input image and its corresponding trimap, we try to extract the foreground from the background. Following is an example: -Input Image: ![](samples/input_images/plant.jpg) -Input Trimap: ![](samples/trimaps/plant.png) -Output alpha Matte: ![](samples/output_mattes/plant_result.jpg) +Input Image: ![](alphamat/samples/input_images/plant.jpg) +Input image should be preferably a RGB image. -This project is implementation of @cite aksoy2017designing . It required implementation of parts of other papers [2,3,4]. +Input Trimap: ![](alphamat/samples/trimaps/plant.png) +The trimap image is a greyscale image that contains information about the foreground(white pixels), background(black pixels) and unknown(grey) pixels. + +Output alpha Matte: ![](alphamat/samples/output_mattes/plant_result.png) +The computed alpha matte is saved as a greyscale image where the pixel values indicate the opacity of the extracted foreground object. These opacity values can be used to blend the foreground object into a diffferent backgound, as shown below: +![](plant_new_backgrounds.jpg) + +Following are some more results. +![](matting_results.jpg) + +The first column is input RGB image, the second column is input trimap, third column is the extracted alpha matte and the last two columns show the foreground object blended on new backgrounds. + +This project is implementation of @cite aksoy2017designing . It also required implementation of parts of other papers [2,3,4]. # Building @@ -33,21 +44,20 @@ Please refer to OpenCV building tutorials for further details, if needed. The built target can be tested as follows: ``` -example_alphamat_information_flow_matting -img= -tri= -out= +/bin/example_alphamat_information_flow_matting -img= -tri= -out= ``` # Source Code of the sample @includelineno alphamat/samples/information_flow_matting.cpp - # References -[1] Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "[Designing Effective Inter-Pixel Information Flow for Natural Image Matting](http://people.inf.ethz.ch/aksoyy/ifm/)", CVPR, 2017. +[1] Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, [Designing Effective Inter-Pixel Information Flow for Natural Image Matting](https://www.researchgate.net/publication/318489370_Designing_Effective_Inter-Pixel_Information_Flow_for_Natural_Image_Matting), CVPR, 2017. -[2] Roweis, Sam T., and Lawrence K. Saul. "[Nonlinear dimensionality reduction by locally linear embedding](https://science.sciencemag.org/content/290/5500/2323)" Science 290.5500 (2000): 2323-2326. +[2] Roweis, Sam T., and Lawrence K. Saul. [Nonlinear dimensionality reduction by locally linear embedding](https://science.sciencemag.org/content/290/5500/2323), Science 290.5500 (2000): 2323-2326. -[3] Anat Levin, Dani Lischinski, Yair Weiss, "[A Closed Form Solution to Natural Image Matting](https://www.researchgate.net/publication/5764820_A_Closed-Form_Solution_to_Natural_Image_Matting)", IEEE TPAMI, 2008. +[3] Anat Levin, Dani Lischinski, Yair Weiss, [A Closed Form Solution to Natural Image Matting](https://www.researchgate.net/publication/5764820_A_Closed-Form_Solution_to_Natural_Image_Matting), IEEE TPAMI, 2008. -[4] Qifeng Chen, Dingzeyu Li, Chi-Keung Tang, "[KNN Matting](http://dingzeyu.li/files/knn-matting-tpami.pdf)", IEEE TPAMI, 2013. +[4] Qifeng Chen, Dingzeyu Li, Chi-Keung Tang, [KNN Matting](http://dingzeyu.li/files/knn-matting-tpami.pdf), IEEE TPAMI, 2013. -[5] Yagiz Aksoy, "[Affinity Based Matting Toolbox](https://github.com/yaksoy/AffinityBasedMattingToolbox)". +[5] Yagiz Aksoy, [Affinity Based Matting Toolbox](https://github.com/yaksoy/AffinityBasedMattingToolbox). diff --git a/modules/alphamat/tutorials/matting_results.jpg b/modules/alphamat/tutorials/matting_results.jpg new file mode 100644 index 00000000000..ebcf5b1d86d Binary files /dev/null and b/modules/alphamat/tutorials/matting_results.jpg differ diff --git a/modules/alphamat/tutorials/plant_new_backgrounds.jpg b/modules/alphamat/tutorials/plant_new_backgrounds.jpg new file mode 100644 index 00000000000..cc8f2b36614 Binary files /dev/null and b/modules/alphamat/tutorials/plant_new_backgrounds.jpg differ diff --git a/modules/cudaarithm/src/lut.cpp b/modules/cudaarithm/src/lut.cpp index a4b4e02650a..5ef28360176 100644 --- a/modules/cudaarithm/src/lut.cpp +++ b/modules/cudaarithm/src/lut.cpp @@ -4,8 +4,6 @@ #include "precomp.hpp" -#include "lut.hpp" - using namespace cv; using namespace cv::cuda; @@ -15,6 +13,9 @@ Ptr cv::cuda::createLookUpTable(InputArray) { throw_no_cuda(); retu #else /* !defined (HAVE_CUDA) || defined (CUDA_DISABLER) */ +// lut.hpp includes cuda_runtime.h and can only be included when we have CUDA +#include "lut.hpp" + Ptr cv::cuda::createLookUpTable(InputArray lut) { return makePtr(lut); diff --git a/modules/cvv/src/stfl/stringutils.cpp b/modules/cvv/src/stfl/stringutils.cpp index 3ce0f145747..71d07d9bb67 100644 --- a/modules/cvv/src/stfl/stringutils.cpp +++ b/modules/cvv/src/stfl/stringutils.cpp @@ -248,15 +248,16 @@ void unescapeCommas(QString &str) QString shortenString(QString &str, int maxLength, bool cutEnd, bool fill) { + const auto horizontalEllipsis = u8"\xE2\x80\xA6"; // u8"…" if (str.size() > maxLength) { if (cutEnd) { - str = str.mid(0, maxLength - 1) + u8"…"; + str = str.mid(0, maxLength - 1) + horizontalEllipsis; } else { - str = u8"…" + + str = horizontalEllipsis + str.mid(str.size() + 1 - maxLength, str.size()); } } diff --git a/modules/cvv/src/util/observer_ptr.hpp b/modules/cvv/src/util/observer_ptr.hpp index fef3feae066..68d1a80f445 100644 --- a/modules/cvv/src/util/observer_ptr.hpp +++ b/modules/cvv/src/util/observer_ptr.hpp @@ -11,6 +11,7 @@ #include //size_t #include // [u]intXX_t #include // since some people like to forget that one +#include namespace cvv { diff --git a/modules/intensity_transform/src/bimef.cpp b/modules/intensity_transform/src/bimef.cpp index 58eac6002af..72555ed9773 100644 --- a/modules/intensity_transform/src/bimef.cpp +++ b/modules/intensity_transform/src/bimef.cpp @@ -234,7 +234,7 @@ static Mat solveLinearEquation(const Mat_& img, Mat_& W_h_, Mat_ tsmooth(const Mat_& src, float lambda=0.01f, float sigma=3.0f, float sharpness=0.001f) diff --git a/modules/julia/CMakeLists.txt b/modules/julia/CMakeLists.txt index 2c47c2b5562..77158cacd7d 100644 --- a/modules/julia/CMakeLists.txt +++ b/modules/julia/CMakeLists.txt @@ -68,6 +68,7 @@ ocv_add_module( opencv_dnn opencv_features2d opencv_objdetect + opencv_calib3d ) set(HDR_PARSER_PATH ${CMAKE_SOURCE_DIR}/modules/python/src2/hdr_parser.py) diff --git a/modules/julia/README.md b/modules/julia/README.md index e19f42fd83c..f460dabdbea 100644 --- a/modules/julia/README.md +++ b/modules/julia/README.md @@ -88,4 +88,4 @@ All other types map directly to the corresponding types on C++. Unlike Python, ` Current Functionality --- -The bindings implement most of the functionality present in the core,imgproc,highgui,videoio,dnn and imgcodecs. The samples also implement some additional manually wrapped functionality. The complete list of automatically wrapped functionality is [here](gen/funclist.csv). +The bindings implement most of the functionality present in the core,imgproc,highgui,videoio,dnn,calib3d and imgcodecs. The samples also implement some additional manually wrapped functionality. The complete list of automatically wrapped functionality is [here](gen/funclist.csv). diff --git a/modules/julia/gen/binding_templates_cpp/cv_core.cpp b/modules/julia/gen/binding_templates_cpp/cv_core.cpp index 05f03f7d4ee..9966ea17747 100644 --- a/modules/julia/gen/binding_templates_cpp/cv_core.cpp +++ b/modules/julia/gen/binding_templates_cpp/cv_core.cpp @@ -41,11 +41,12 @@ typedef ParameterList> type; // #ifdef HAVE_OPENCV_FEATURES2D - template <> - struct SuperType - { - typedef cv::Algorithm type; - }; + // template <> + // struct SuperType + // { + // typedef cv::Algorithm type; + // }; + // TODO: Needs to be fixed but doesn't matter for now template <> struct SuperType { @@ -88,7 +89,6 @@ JLCXX_MODULE cv_wrap(jlcxx::Module &mod) - ${cpp_code} // // Manual Wrapping BEGIN @@ -128,16 +128,21 @@ JLCXX_MODULE cv_wrap(jlcxx::Module &mod) #endif #ifdef HAVE_OPENCV_FEATURES2D - mod.add_type("Feature2D", jlcxx::julia_base_type()); + mod.add_type("Feature2D"); mod.add_type("SimpleBlobDetector", jlcxx::julia_base_type()); mod.add_type("SimpleBlobDetector_Params"); - - mod.method("jlopencv_cv_cv_Feature2D_cv_Feature2D_detect", [](cv::Ptr &cobj, Mat &image, Mat &mask) {vector keypoints; cobj->detect(image, keypoints, mask); return keypoints; }); - mod.method("jlopencv_cv_cv_SimpleBlobDetector_create", [](SimpleBlobDetector_Params ¶meters) { auto retval = cv::SimpleBlobDetector::create(parameters); return retval; }); #endif // // Manual Wrapping END // + ${cpp_code} + +#ifdef HAVE_OPENCV_FEATURES2D + + mod.method("jlopencv_cv_cv_Feature2D_cv_Feature2D_detect", [](cv::Ptr &cobj, Mat &image, Mat &mask) {vector keypoints; cobj->detect(image, keypoints, mask); return keypoints; }); + mod.method("jlopencv_cv_cv_SimpleBlobDetector_create", [](SimpleBlobDetector_Params ¶meters) { auto retval = cv::SimpleBlobDetector::create(parameters); return retval; }); +#endif + } diff --git a/modules/julia/gen/cpp_files/jlcv.hpp b/modules/julia/gen/cpp_files/jlcv.hpp index 7fa9ea11b05..a9769c462c5 100644 --- a/modules/julia/gen/cpp_files/jlcv.hpp +++ b/modules/julia/gen/cpp_files/jlcv.hpp @@ -75,6 +75,11 @@ typedef cv::dnn::DictValue LayerId; typedef cv::dnn::Backend dnn_Backend; #endif +#ifdef HAVE_OPENCV_CALIB3D + +#include +#endif + template struct get_template_type; template diff --git a/modules/julia/gen/defval.txt b/modules/julia/gen/defval.txt index c1313a3d5fc..5d65465203a 100644 --- a/modules/julia/gen/defval.txt +++ b/modules/julia/gen/defval.txt @@ -65,4 +65,38 @@ Int32|0|0 Float64|255.|255 Scalar|Scalar(1)|cpp_to_julia(ScalarOXP()) Int32|1|1 -Size{Int32}|Size()|cpp_to_julia(SizeOP()) \ No newline at end of file +Size{Int32}|Size()|cpp_to_julia(SizeOP()) +TermCriteria|TermCriteria(TermCriteria::EPS + TermCriteria::COUNT, 20, FLT_EPSILON)|cpp_to_julia(TermCriteriaOTermCriteriaggEPSGRGTermCriteriaggCOUNTSGYWSGFLTREPSILONP()) +Int32|RANSAC|cv_RANSAC +Float32|8.0|8.0 +Float64|-1|-1 +Int32|21|21 +TermCriteria|TermCriteria( TermCriteria::COUNT + TermCriteria::EPS, 30, DBL_EPSILON)|TermCriteriaOGTermCriteriaggCOUNTGRGTermCriteriaggEPSSGZWSGDBLREPSILONP +TermCriteria|TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 1e-6)|TermCriteriaOTermCriteriaggCOUNTRTermCriteriaggEPSSGZWSGXeTcP +Int32|CALIB_CB_SYMMETRIC_GRID|cv_CALIB_CB_SYMMETRIC_GRID +InputArray|cv::Mat()|CxxMat() +Int32|SOLVEPNP_ITERATIVE|cv_SOLVEPNP_ITERATIVE +Float64|3|3 +Int32|CALIB_FIX_INTRINSIC|cv_CALIB_FIX_INTRINSIC +Float64|5|5 +Float64|0.99|0.99 +Int32|CALIB_ZERO_DISPARITY|cv_CALIB_ZERO_DISPARITY +size_t|2000|2000 +SolvePnPMethod|SOLVEPNP_ITERATIVE|cv_SOLVEPNP_ITERATIVE +Float64|0.0|0.0 +Ptr{Feature2D}|SimpleBlobDetector::create()|SimpleBlobDetectorggcreateOP +Int32|StereoSGBM::MODE_SGBM|StereoSGBMggMODERSGBM +Int32|CALIB_CB_ADAPTIVE_THRESH + CALIB_CB_NORMALIZE_IMAGE|cv_CALIB_CB_ADAPTIVE_THRESH + cv_CALIB_CB_NORMALIZE_IMAGE +Float64|3.|3 +size_t|10|10 +Int32|16|16 +Point{Float64}|Point2d(0, 0)|PointYdOWSGWP +Int32|2000|2000 +Int32|FM_RANSAC|cv_FM_RANSAC +Int32|100|100 +TermCriteria|TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100, DBL_EPSILON)|TermCriteriaOTermCriteriaggCOUNTGRGTermCriteriaggEPSSGXWWSGDBLREPSILONP +HandEyeCalibrationMethod|CALIB_HAND_EYE_TSAI|cv_CALIB_HAND_EYE_TSAI +Float32|0.8F|0.8 +Int32|fisheye::CALIB_FIX_INTRINSIC|cv_fisheye_CALIB_FIX_INTRINSIC +Float64|0.999|0.999 +Float64|0.995|0.995 \ No newline at end of file diff --git a/modules/julia/gen/funclist.csv b/modules/julia/gen/funclist.csv index ad8a804f048..5c43e5cab23 100644 --- a/modules/julia/gen/funclist.csv +++ b/modules/julia/gen/funclist.csv @@ -373,4 +373,120 @@ cv.setTrackbarMax cv.setTrackbarMin cv.addText cv.displayOverlay -cv.displayStatusBar \ No newline at end of file +cv.displayStatusBar +cv.Rodrigues +cv.findHomography +cv.RQDecomp3x3 +cv.decomposeProjectionMatrix +cv.matMulDeriv +cv.composeRT +cv.projectPoints +cv.solvePnP +cv.solvePnPRansac +cv.solveP3P +cv.solvePnPRefineLM +cv.solvePnPRefineVVS +cv.solvePnPGeneric +cv.initCameraMatrix2D +cv.findChessboardCorners +cv.checkChessboard +cv.findChessboardCornersSB +cv.findChessboardCornersSB +cv.estimateChessboardSharpness +cv.find4QuadCornerSubpix +cv.drawChessboardCorners +cv.drawFrameAxes +cv.CirclesGridFinderParameters.CirclesGridFinderParameters +cv.findCirclesGrid +cv.findCirclesGrid +cv.calibrateCamera +cv.calibrateCamera +cv.calibrateCameraRO +cv.calibrateCameraRO +cv.calibrationMatrixValues +cv.stereoCalibrate +cv.stereoCalibrate +cv.stereoRectify +cv.stereoRectifyUncalibrated +cv.rectify3Collinear +cv.getOptimalNewCameraMatrix +cv.calibrateHandEye +cv.convertPointsToHomogeneous +cv.convertPointsFromHomogeneous +cv.findFundamentalMat +cv.findFundamentalMat +cv.findEssentialMat +cv.findEssentialMat +cv.decomposeEssentialMat +cv.recoverPose +cv.recoverPose +cv.recoverPose +cv.computeCorrespondEpilines +cv.triangulatePoints +cv.correctMatches +cv.filterSpeckles +cv.getValidDisparityROI +cv.validateDisparity +cv.reprojectImageTo3D +cv.sampsonDistance +cv.estimateAffine3D +cv.estimateTranslation3D +cv.estimateAffine2D +cv.estimateAffinePartial2D +cv.decomposeHomographyMat +cv.filterHomographyDecompByVisibleRefpoints +cv.StereoMatcher.compute +cv.StereoMatcher.getMinDisparity +cv.StereoMatcher.setMinDisparity +cv.StereoMatcher.getNumDisparities +cv.StereoMatcher.setNumDisparities +cv.StereoMatcher.getBlockSize +cv.StereoMatcher.setBlockSize +cv.StereoMatcher.getSpeckleWindowSize +cv.StereoMatcher.setSpeckleWindowSize +cv.StereoMatcher.getSpeckleRange +cv.StereoMatcher.setSpeckleRange +cv.StereoMatcher.getDisp12MaxDiff +cv.StereoMatcher.setDisp12MaxDiff +cv.StereoBM.getPreFilterType +cv.StereoBM.setPreFilterType +cv.StereoBM.getPreFilterSize +cv.StereoBM.setPreFilterSize +cv.StereoBM.getPreFilterCap +cv.StereoBM.setPreFilterCap +cv.StereoBM.getTextureThreshold +cv.StereoBM.setTextureThreshold +cv.StereoBM.getUniquenessRatio +cv.StereoBM.setUniquenessRatio +cv.StereoBM.getSmallerBlockSize +cv.StereoBM.setSmallerBlockSize +cv.StereoBM.getROI1 +cv.StereoBM.setROI1 +cv.StereoBM.getROI2 +cv.StereoBM.setROI2 +cv.StereoBM.create +cv.StereoSGBM.getPreFilterCap +cv.StereoSGBM.setPreFilterCap +cv.StereoSGBM.getUniquenessRatio +cv.StereoSGBM.setUniquenessRatio +cv.StereoSGBM.getP1 +cv.StereoSGBM.setP1 +cv.StereoSGBM.getP2 +cv.StereoSGBM.setP2 +cv.StereoSGBM.getMode +cv.StereoSGBM.setMode +cv.StereoSGBM.create +cv.undistort +cv.initUndistortRectifyMap +cv.getDefaultNewCameraMatrix +cv.undistortPoints +cv.undistortPoints +cv.fisheye.projectPoints +cv.fisheye.distortPoints +cv.fisheye.undistortPoints +cv.fisheye.initUndistortRectifyMap +cv.fisheye.undistortImage +cv.fisheye.estimateNewCameraMatrixForUndistortRectify +cv.fisheye.calibrate +cv.fisheye.stereoRectify +cv.fisheye.stereoCalibrate diff --git a/modules/julia/gen/gen3_julia_cxx.py b/modules/julia/gen/gen3_julia_cxx.py old mode 100644 new mode 100755 index 7dec06ab48f..77a2a15748d --- a/modules/julia/gen/gen3_julia_cxx.py +++ b/modules/julia/gen/gen3_julia_cxx.py @@ -50,21 +50,6 @@ def handle_def_arg(inp, tp = '', ns=''): inp = inp.strip() out = '' - if tp in julia_types: - out = inp - elif not inp or inp=='Mat()': - if tp=='Mat' or tp=='InputArray': - out= 'CxxMat()' - out = tp+'()' - - elif inp=="String()": - out= '""' - - elif '(' in inp or ':' in inp: - out = "cpp_to_julia("+get_var(inp)+"())" - - else: - print("Default not found") if inp in jl_cpp_defmap[tp]: out = jl_cpp_defmap[tp][inp] @@ -129,8 +114,11 @@ def get_argument_full(self, classname='', isalgo = False): def get_argument_opt(self, ns=''): # [print(arg.default_value,":",handle_def_arg(arg.default_value, handle_jl_arg(arg.tp))) for arg in self.optlist] - str2 = ", ".join(["%s::%s = %s(%s)" % (arg.name, handle_jl_arg(arg.tp), handle_jl_arg(arg.tp) if (arg.tp == 'int' or arg.tp=='float' or arg.tp=='double') else '', handle_def_arg(arg.default_value, handle_jl_arg(arg.tp), ns)) for arg in self.optlist]) - return str2 + try: + str2 = ", ".join(["%s::%s = %s(%s)" % (arg.name, handle_jl_arg(arg.tp), handle_jl_arg(arg.tp) if (arg.tp == 'int' or arg.tp=='float' or arg.tp=='double') else '', handle_def_arg(arg.default_value, handle_jl_arg(arg.tp), ns)) for arg in self.optlist]) + return str2 + except KeyError: + return '' def get_argument_def(self, classname, isalgo): arglist = self.inlist diff --git a/modules/julia/gen/gen_all.py b/modules/julia/gen/gen_all.py index 701d4a91581..494c9e703be 100644 --- a/modules/julia/gen/gen_all.py +++ b/modules/julia/gen/gen_all.py @@ -32,6 +32,8 @@ hdr_list.append(mod_path+"/videoio/include/opencv2/videoio.hpp") elif module =='opencv_highgui': hdr_list.append(mod_path+"/highgui/include/opencv2/highgui.hpp") + elif module =='opencv_calib3d': + hdr_list.append(mod_path+"/calib3d/include/opencv2/calib3d.hpp") if not os.path.exists('autogen_cpp'): os.makedirs('autogen_cpp') diff --git a/modules/julia/gen/typemap.txt b/modules/julia/gen/typemap.txt index e1750ed6e90..df5e78e3270 100644 --- a/modules/julia/gen/typemap.txt +++ b/modules/julia/gen/typemap.txt @@ -45,3 +45,8 @@ vector:Array{Array{Int32, 1}, 1} float:Float32 Ptr:Ptr{Float32} vector:Array{Vec{Float32, 6}, 1} +Ptr:Ptr{Feature2D} +Point2d:Point{Float64} +SolvePnPMethod:SolvePnPMethod +CirclesGridFinderParameters:CirclesGridFinderParameters +HandEyeCalibrationMethod:HandEyeCalibrationMethod diff --git a/modules/julia/samples/chessboard_corners.jl b/modules/julia/samples/chessboard_corners.jl new file mode 100644 index 00000000000..cc9a1e7b8dd --- /dev/null +++ b/modules/julia/samples/chessboard_corners.jl @@ -0,0 +1,20 @@ +using OpenCV + +const cv = OpenCV + + +# chess1.png is at https://raw.githubusercontent.com/opencv/opencv_extra/master/testdata/cv/cameracalibration/chess1.png +img = cv.imread("chess1.png",cv.IMREAD_GRAYSCALE) +climg = cv.cvtColor(img, cv.COLOR_GRAY2BGR) + +# Find the chess board corners +ret, corners = cv.findChessboardCorners(img, cv.Size{Int32}(7,5)) + +# If found, add object points, image points (after refining them) +if ret + climg = cv.drawChessboardCorners(climg, cv.Size{Int32}(7,5), corners,ret) + cv.imshow("img",climg) + cv.waitKey(Int32(0)) + + cv.destroyAllWindows() +end diff --git a/modules/julia/tutorials/julia.markdown b/modules/julia/tutorials/julia.markdown index 2e7a06031e4..f216daac4ea 100644 --- a/modules/julia/tutorials/julia.markdown +++ b/modules/julia/tutorials/julia.markdown @@ -16,7 +16,7 @@ Inspite of all this, Julia severely lacks in a lot of traditional computer visio The Bindings ----------------------- -The OpenCV bindings for Julia are created automatically using Python scripts at configure time and then installed with the Julia package manager on the system. These bindings cover most of the important functionality present in the core, imgproc, imgcodecs, highgui, videio, and dnn modules. These bindings depend on CxxWrap.jl and the process for usage and compilation is explained in detail below. The Bindings have been tested on Ubuntu and Mac. Windows might work but is not officially tested and supported right now. +The OpenCV bindings for Julia are created automatically using Python scripts at configure time and then installed with the Julia package manager on the system. These bindings cover most of the important functionality present in the core, imgproc, imgcodecs, highgui, videio, calib3d, and dnn modules. These bindings depend on CxxWrap.jl and the process for usage and compilation is explained in detail below. The Bindings have been tested on Ubuntu and Mac. Windows might work but is not officially tested and supported right now. The generation process and the method by which the binding works are similar to the Python bindings. The only major difference is that CxxWrap.jl does not support optional arguments. As a consequence, it's necessary to define the optional arguments in Julia code which adds a lot of additional complexity. diff --git a/modules/mcc/CMakeLists.txt b/modules/mcc/CMakeLists.txt index 806303505f3..2b7fadbaa14 100644 --- a/modules/mcc/CMakeLists.txt +++ b/modules/mcc/CMakeLists.txt @@ -1,2 +1,9 @@ set(the_description "Macbeth Chart Detection") -ocv_define_module(mcc opencv_core opencv_imgproc opencv_calib3d opencv_photo opencv_dnn WRAP python) +ocv_define_module(mcc + opencv_core + opencv_imgproc + opencv_calib3d + opencv_dnn + WRAP + python +) diff --git a/modules/mcc/include/opencv2/mcc.hpp b/modules/mcc/include/opencv2/mcc.hpp index 2027485fb2a..e49f3053ffa 100644 --- a/modules/mcc/include/opencv2/mcc.hpp +++ b/modules/mcc/include/opencv2/mcc.hpp @@ -32,8 +32,15 @@ #include "mcc/checker_detector.hpp" #include "mcc/checker_model.hpp" +#include "mcc/ccm.hpp" /** @defgroup mcc Macbeth Chart module +@{ + @defgroup color_correction Color Correction Model +@} + + +@addtogroup mcc Introduction ------------ @@ -47,7 +54,6 @@ colors that are present in the image, based on this information we can apply any suitable algorithm to find the actual color of all the objects present in the image. - */ #endif diff --git a/modules/mcc/include/opencv2/mcc/ccm.hpp b/modules/mcc/include/opencv2/mcc/ccm.hpp new file mode 100644 index 00000000000..a3686db473d --- /dev/null +++ b/modules/mcc/include/opencv2/mcc/ccm.hpp @@ -0,0 +1,519 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_CCM_HPP__ +#define __OPENCV_MCC_CCM_HPP__ + +#include +#include + +namespace cv +{ +namespace ccm +{ +/** @addtogroup color_correction +@{ + +Introduction +------------ + +The purpose of color correction is to adjust the color response of input +and output devices to a known state. The device being calibrated is sometimes +called the calibration source; the color space used as the standard is sometimes +called the calibration target. Color calibration has been used in many industries, +such as television production, games, photography, engineering, chemistry, +medicine, etc. Due to the manufacturing process of the input and output equipment, +the channel response has nonlinear distortion. In order to correct the picture output +of the equipment, it is nessary to calibrate the captured color and the actual color. + +*/ + + + +/** @brief Enum of the possible types of ccm. +*/ +enum CCM_TYPE +{ + CCM_3x3, ///< The CCM with the shape \f$3\times3\f$ performs linear transformation on color values. + CCM_4x3, ///< The CCM with the shape \f$4\times3\f$ performs affine transformation. +}; + +/** @brief Enum of the possible types of initial method. +*/ +enum INITIAL_METHOD_TYPE +{ + INITIAL_METHOD_WHITE_BALANCE, ///< The white balance method. The initial value is:\n + /// \f$ + /// M_{CCM}= + /// \begin{bmatrix} + /// k_R & 0 & 0\\ + /// 0 & k_G & 0\\ + /// 0 & 0 & k_B\\ + /// \end{bmatrix} + /// \f$\n + /// where\n + /// \f$ + /// k_R=mean(R_{li}')/mean(R_{li})\\ + /// k_R=mean(G_{li}')/mean(G_{li})\\ + /// k_R=mean(B_{li}')/mean(B_{li}) + /// \f$ + INITIAL_METHOD_LEAST_SQUARE, ///0\\ +C_{sl}=0, \qquad C_s=0 +\f] + +Because \f$exp(ln(0))\to\infty \f$, the channel whose component is 0 is directly mapped to 0 in the formula above. + +For fitting channels respectively, we have: +\f[ +r=polyfit(ln(R_s),ln(R_{dl}))\\ +g=polyfit(ln(G_s),ln(G_{dl}))\\ +b=polyfit(ln(B_s),ln(B_{dl}))\\ +\f] +Note that the parameter of \f$ln(*) \f$ cannot be 0. +Therefore, we need to delete the channels whose values are 0 from \f$R_s \f$ and \f$R_{dl} \f$, \f$G_s\f$ and \f$G_{dl}\f$, \f$B_s\f$ and \f$B_{dl}\f$. + +Therefore: + +\f[ +ln(R_{sl})=r(ln(R_s)), \qquad R_s>0\\ +R_{sl}=0, \qquad R_s=0\\ +ln(G_{sl})=g(ln(G_s)),\qquad G_s>0\\ +G_{sl}=0, \qquad G_s=0\\ +ln(B_{sl})=b(ln(B_s)),\qquad B_s>0\\ +B_{sl}=0, \qquad B_s=0\\ +\f] + +For grayscale polynomials, there are also: +\f[ +f=polyfit(ln(G_{sl}),ln(G_{dl})) +\f] +and: +\f[ +ln(C_{sl})=f(ln(C_s)), \qquad C_s>0\\ +C_sl=0, \qquad C_s=0 +\f] +*/ +enum LINEAR_TYPE +{ + + LINEARIZATION_IDENTITY, /// p; +}; + +//! @} ccm +} // namespace ccm +} // namespace cv + +#endif \ No newline at end of file diff --git a/modules/mcc/samples/color_correction_model.cpp b/modules/mcc/samples/color_correction_model.cpp new file mode 100644 index 00000000000..20df704370e --- /dev/null +++ b/modules/mcc/samples/color_correction_model.cpp @@ -0,0 +1,158 @@ +//! [tutorial] +#include + +#include +#include +#include +#include + +using namespace std; +using namespace cv; +using namespace mcc; +using namespace ccm; +using namespace std; + +const char *about = "Basic chart detection"; +const char *keys = + "{ help h | | show this message }" + "{t | | chartType: 0-Standard, 1-DigitalSG, 2-Vinyl }" + "{v | | Input from video file, if ommited, input comes from camera }" + "{ci | 0 | Camera id if input doesnt come from video (-v) }" + "{f | 1 | Path of the file to process (-v) }" + "{nc | 1 | Maximum number of charts in the image }"; + +int main(int argc, char *argv[]) +{ + + + // ---------------------------------------------------------- + // Scroll down a bit (~40 lines) to find actual relevant code + // ---------------------------------------------------------- + //! [get_messages_of_image] + CommandLineParser parser(argc, argv, keys); + parser.about(about); + if (argc==1 || parser.has("help")) + { + parser.printMessage(); + return 0; + } + + int t = parser.get("t"); + int nc = parser.get("nc"); + string filepath = parser.get("f"); + + CV_Assert(0 <= t && t <= 2); + TYPECHART chartType = TYPECHART(t); + + + if (!parser.check()) + { + parser.printErrors(); + return 0; + } + + Mat image = imread(filepath, IMREAD_COLOR); + if (!image.data) + { + cout << "Invalid Image!" << endl; + return 1; + } + //! [get_messages_of_image] + + Mat imageCopy = image.clone(); + Ptr detector = CCheckerDetector::create(); + // Marker type to detect + if (!detector->process(image, chartType, nc)) + { + printf("ChartColor not detected \n"); + return 2; + } + //! [get_color_checker] + vector> checkers = detector->getListColorChecker(); + //! [get_color_checker] + for (Ptr checker : checkers) + { + //! [create] + Ptr cdraw = CCheckerDraw::create(checker); + cdraw->draw(image); + Mat chartsRGB = checker->getChartsRGB(); + Mat src = chartsRGB.col(1).clone().reshape(3, chartsRGB.rows/3); + src /= 255.0; + //! [create] + + //compte color correction matrix + //! [get_ccm_Matrix] + ColorCorrectionModel model1(src, COLORCHECKER_Vinyl); + model1.run(); + Mat ccm = model1.getCCM(); + std::cout<<"ccm "<(18, 1) << + // Vec3d(100, 0.00520000001, -0.0104), + // Vec3d(73.0833969, -0.819999993, -2.02099991), + // Vec3d(62.493, 0.425999999, -2.23099995), + // Vec3d(50.4640007, 0.446999997, -2.32399988), + // Vec3d(37.7970009, 0.0359999985, -1.29700005), + // Vec3d(0, 0, 0), + // Vec3d(51.5880013, 73.5179977, 51.5690002), + // Vec3d(93.6989975, -15.7340002, 91.9420013), + // Vec3d(69.4079971, -46.5940018, 50.4869995), + // Vec3d(66.61000060000001, -13.6789999, -43.1720009), + // Vec3d(11.7110004, 16.9799995, -37.1759987), + // Vec3d(51.973999, 81.9440002, -8.40699959), + // Vec3d(40.5489998, 50.4399986, 24.8490009), + // Vec3d(60.8160019, 26.0690002, 49.4420013), + // Vec3d(52.2529984, -19.9500008, -23.9960003), + // Vec3d(51.2859993, 48.4700012, -15.0579996), + // Vec3d(68.70700069999999, 12.2959995, 16.2129993), + // Vec3d(63.6839981, 10.2930002, 16.7639999)); + + // ColorCorrectionModel model8(src,ref,COLOR_SPACE_Lab_D50_2); + // model8.run(); + //! [reference_color_values] + + //! [make_color_correction] + Mat img_; + cvtColor(image, img_, COLOR_BGR2RGB); + img_.convertTo(img_, CV_64F); + const int inp_size = 255; + const int out_size = 255; + img_ = img_ / inp_size; + Mat calibratedImage= model1.infer(img_); + Mat out_ = calibratedImage * out_size; + //! [make_color_correction] + + //! [Save_calibrated_image] + // Save the calibrated image to {FILE_NAME}.calibrated.{FILE_EXT} + out_.convertTo(out_, CV_8UC3); + Mat img_out = min(max(out_, 0), out_size); + Mat out_img; + cvtColor(img_out, out_img, COLOR_RGB2BGR); + string filename = filepath.substr(filepath.find_last_of('/')+1); + size_t dotIndex = filename.find_last_of('.'); + string baseName = filename.substr(0, dotIndex); + string ext = filename.substr(dotIndex+1, filename.length()-dotIndex); + string calibratedFilePath = baseName + ".calibrated." + ext; + imwrite(calibratedFilePath, out_img); + //! [Save_calibrated_image] + + } + + return 0; +} +//! [tutorial] \ No newline at end of file diff --git a/modules/mcc/src/ccm.cpp b/modules/mcc/src/ccm.cpp new file mode 100644 index 00000000000..7e26d164124 --- /dev/null +++ b/modules/mcc/src/ccm.cpp @@ -0,0 +1,438 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "opencv2/mcc/ccm.hpp" +#include "linearize.hpp" +namespace cv { +namespace ccm { +class ColorCorrectionModel::Impl +{ +public: + Mat src; + std::shared_ptr dst = std::make_shared(); + Mat dist; + RGBBase_& cs; + Mat mask; + + // RGBl of detected data and the reference + Mat src_rgbl; + Mat dst_rgbl; + + // ccm type and shape + CCM_TYPE ccm_type; + int shape; + + // linear method and distance + std::shared_ptr linear = std::make_shared(); + DISTANCE_TYPE distance; + LINEAR_TYPE linear_type; + + Mat weights; + Mat weights_list; + Mat ccm; + Mat ccm0; + double gamma; + int deg; + std::vector saturated_threshold; + INITIAL_METHOD_TYPE initial_method_type; + double weights_coeff; + int masked_len; + double loss; + int max_count; + double epsilon; + Impl(); + + /** @brief Make no change for CCM_3x3. + convert cv::Mat A to [A, 1] in CCM_4x3. + @param inp the input array, type of cv::Mat. + @return the output array, type of cv::Mat + */ + Mat prepare(const Mat& inp); + + /** @brief Calculate weights and mask. + @param weights_list the input array, type of cv::Mat. + @param weights_coeff type of double. + @param saturate_mask the input array, type of cv::Mat. + */ + void calWeightsMasks(const Mat& weights_list, double weights_coeff, Mat saturate_mask); + + /** @brief Fitting nonlinear - optimization initial value by white balance. + @return the output array, type of Mat + */ + void initialWhiteBalance(void); + + /** @brief Fitting nonlinear-optimization initial value by least square. + @param fit if fit is True, return optimalization for rgbl distance function. + */ + void initialLeastSquare(bool fit = false); + + double calc_loss_(Color color); + double calc_loss(const Mat ccm_); + + /** @brief Fitting ccm if distance function is associated with CIE Lab color space. + see details in https://github.com/opencv/opencv/blob/master/modules/core/include/opencv2/core/optim.hpp + Set terminal criteria for solver is possible. + */ + void fitting(void); + + void getColor(Mat& img_, bool islinear = false); + void getColor(CONST_COLOR constcolor); + void getColor(Mat colors_, COLOR_SPACE cs_, Mat colored_); + void getColor(Mat colors_, COLOR_SPACE ref_cs_); + + /** @brief Loss function base on cv::MinProblemSolver::Function. + see details in https://github.com/opencv/opencv/blob/master/modules/core/include/opencv2/core/optim.hpp + */ + class LossFunction : public MinProblemSolver::Function + { + public: + ColorCorrectionModel::Impl* ccm_loss; + LossFunction(ColorCorrectionModel::Impl* ccm) + : ccm_loss(ccm) {}; + + /** @brief Reset dims to ccm->shape. + */ + int getDims() const CV_OVERRIDE + { + return ccm_loss->shape; + } + + /** @brief Reset calculation. + */ + double calc(const double* x) const CV_OVERRIDE + { + Mat ccm_(ccm_loss->shape, 1, CV_64F); + for (int i = 0; i < ccm_loss->shape; i++) + { + ccm_.at(i, 0) = x[i]; + } + ccm_ = ccm_.reshape(0, ccm_loss->shape / 3); + return ccm_loss->calc_loss(ccm_); + } + }; +}; + +ColorCorrectionModel::Impl::Impl() + : cs(*GetCS::getInstance().get_rgb(COLOR_SPACE_sRGB)) + , ccm_type(CCM_3x3) + , distance(DISTANCE_CIE2000) + , linear_type(LINEARIZATION_GAMMA) + , weights(Mat()) + , gamma(2.2) + , deg(3) + , saturated_threshold({ 0, 0.98 }) + , initial_method_type(INITIAL_METHOD_LEAST_SQUARE) + , weights_coeff(0) + , max_count(5000) + , epsilon(1.e-4) +{} + +Mat ColorCorrectionModel::Impl::prepare(const Mat& inp) +{ + switch (ccm_type) + { + case cv::ccm::CCM_3x3: + shape = 9; + return inp; + case cv::ccm::CCM_4x3: + { + shape = 12; + Mat arr1 = Mat::ones(inp.size(), CV_64F); + Mat arr_out(inp.size(), CV_64FC4); + Mat arr_channels[3]; + split(inp, arr_channels); + merge(std::vector { arr_channels[0], arr_channels[1], arr_channels[2], arr1 }, arr_out); + return arr_out; + } + default: + CV_Error(Error::StsBadArg, "Wrong ccm_type!"); + break; + } +} + +void ColorCorrectionModel::Impl::calWeightsMasks(const Mat& weights_list_, double weights_coeff_, Mat saturate_mask) +{ + // weights + if (!weights_list_.empty()) + { + weights = weights_list_; + } + else if (weights_coeff_ != 0) + { + pow(dst->toLuminant(cs.io), weights_coeff_, weights); + } + + // masks + Mat weight_mask = Mat::ones(src.rows, 1, CV_8U); + if (!weights.empty()) + { + weight_mask = weights > 0; + } + this->mask = (weight_mask) & (saturate_mask); + + // weights' mask + if (!weights.empty()) + { + Mat weights_masked = maskCopyTo(this->weights, this->mask); + weights = weights_masked / mean(weights_masked)[0]; + } + masked_len = (int)sum(mask)[0]; +} + +void ColorCorrectionModel::Impl::initialWhiteBalance(void) +{ + Mat schannels[4]; + split(src_rgbl, schannels); + Mat dchannels[4]; + split(dst_rgbl, dchannels); + std::vector initial_vec = { sum(dchannels[0])[0] / sum(schannels[0])[0], 0, 0, 0, + sum(dchannels[1])[0] / sum(schannels[1])[0], 0, 0, 0, + sum(dchannels[2])[0] / sum(schannels[2])[0], 0, 0, 0 }; + std::vector initial_vec_(initial_vec.begin(), initial_vec.begin() + shape); + Mat initial_white_balance = Mat(initial_vec_, true).reshape(0, shape / 3); + ccm0 = initial_white_balance; +} + +void ColorCorrectionModel::Impl::initialLeastSquare(bool fit) +{ + Mat A, B, w; + if (weights.empty()) + { + A = src_rgbl; + B = dst_rgbl; + } + else + { + pow(weights, 0.5, w); + Mat w_; + merge(std::vector { w, w, w }, w_); + A = w_.mul(src_rgbl); + B = w_.mul(dst_rgbl); + } + solve(A.reshape(1, A.rows), B.reshape(1, B.rows), ccm0, DECOMP_SVD); + + // if fit is True, return optimalization for rgbl distance function. + if (fit) + { + ccm = ccm0; + Mat residual = A.reshape(1, A.rows) * ccm.reshape(0, shape / 3) - B.reshape(1, B.rows); + Scalar s = residual.dot(residual); + double sum = s[0]; + loss = sqrt(sum / masked_len); + } +} + +double ColorCorrectionModel::Impl::calc_loss_(Color color) +{ + Mat distlist = color.diff(*dst, distance); + Color lab = color.to(COLOR_SPACE_Lab_D50_2); + Mat dist_; + pow(distlist, 2, dist_); + if (!weights.empty()) + { + dist_ = weights.mul(dist_); + } + Scalar ss = sum(dist_); + return ss[0]; +} + +double ColorCorrectionModel::Impl::calc_loss(const Mat ccm_) +{ + Mat converted = src_rgbl.reshape(1, 0) * ccm_; + Color color(converted.reshape(3, 0), *(cs.l)); + return calc_loss_(color); +} + +void ColorCorrectionModel::Impl::fitting(void) +{ + cv::Ptr solver = cv::DownhillSolver::create(); + cv::Ptr ptr_F(new LossFunction(this)); + solver->setFunction(ptr_F); + Mat reshapeccm = ccm0.clone().reshape(0, 1); + Mat step = Mat::ones(reshapeccm.size(), CV_64F); + solver->setInitStep(step); + TermCriteria termcrit = TermCriteria(TermCriteria::MAX_ITER + TermCriteria::EPS, max_count, epsilon); + solver->setTermCriteria(termcrit); + double res = solver->minimize(reshapeccm); + ccm = reshapeccm.reshape(0, shape / 3); + loss = pow((res / masked_len), 0.5); +} + +Mat ColorCorrectionModel::infer(const Mat& img, bool islinear) +{ + if (!p->ccm.data) + { + CV_Error(Error::StsBadArg, "No CCM values!" ); + } + Mat img_lin = (p->linear)->linearize(img); + Mat img_ccm(img_lin.size(), img_lin.type()); + Mat ccm_ = p->ccm.reshape(0, p->shape / 3); + img_ccm = multiple(p->prepare(img_lin), ccm_); + if (islinear == true) + { + return img_ccm; + } + return p->cs.fromL(img_ccm); +} + +void ColorCorrectionModel::Impl::getColor(CONST_COLOR constcolor) +{ + dst = (GetColor::getColor(constcolor)); +} +void ColorCorrectionModel::Impl::getColor(Mat colors_, COLOR_SPACE ref_cs_) +{ + dst.reset(new Color(colors_, *GetCS::getInstance().get_cs(ref_cs_))); +} +void ColorCorrectionModel::Impl::getColor(Mat colors_, COLOR_SPACE cs_, Mat colored_) +{ + dst.reset(new Color(colors_, *GetCS::getInstance().get_cs(cs_), colored_)); +} +ColorCorrectionModel::ColorCorrectionModel(const Mat& src_, CONST_COLOR constcolor) + : p(std::make_shared()) +{ + p->src = src_; + p->getColor(constcolor); +} +ColorCorrectionModel::ColorCorrectionModel(const Mat& src_, Mat colors_, COLOR_SPACE ref_cs_) + : p(std::make_shared()) +{ + p->src = src_; + p->getColor(colors_, ref_cs_); +} +ColorCorrectionModel::ColorCorrectionModel(const Mat& src_, Mat colors_, COLOR_SPACE cs_, Mat colored_) + : p(std::make_shared()) +{ + p->src = src_; + p->getColor(colors_, cs_, colored_); +} + +void ColorCorrectionModel::setColorSpace(COLOR_SPACE cs_) +{ + p->cs = *GetCS::getInstance().get_rgb(cs_); +} +void ColorCorrectionModel::setCCM_TYPE(CCM_TYPE ccm_type_) +{ + p->ccm_type = ccm_type_; +} +void ColorCorrectionModel::setDistance(DISTANCE_TYPE distance_) +{ + p->distance = distance_; +} +void ColorCorrectionModel::setLinear(LINEAR_TYPE linear_type) +{ + p->linear_type = linear_type; +} +void ColorCorrectionModel::setLinearGamma(const double& gamma) +{ + p->gamma = gamma; +} +void ColorCorrectionModel::setLinearDegree(const int& deg) +{ + p->deg = deg; +} +void ColorCorrectionModel::setSaturatedThreshold(const double& lower, const double& upper) +{ //std::vector saturated_threshold + p->saturated_threshold = { lower, upper }; +} +void ColorCorrectionModel::setWeightsList(const Mat& weights_list) +{ + p->weights_list = weights_list; +} +void ColorCorrectionModel::setWeightCoeff(const double& weights_coeff) +{ + p->weights_coeff = weights_coeff; +} +void ColorCorrectionModel::setInitialMethod(INITIAL_METHOD_TYPE initial_method_type) +{ + p->initial_method_type = initial_method_type; +} +void ColorCorrectionModel::setMaxCount(const int& max_count_) +{ + p->max_count = max_count_; +} +void ColorCorrectionModel::setEpsilon(const double& epsilon_) +{ + p->epsilon = epsilon_; +} +void ColorCorrectionModel::run() +{ + + Mat saturate_mask = saturate(p->src, p->saturated_threshold[0], p->saturated_threshold[1]); + p->linear = getLinear(p->gamma, p->deg, p->src, *(p->dst), saturate_mask, (p->cs), p->linear_type); + p->calWeightsMasks(p->weights_list, p->weights_coeff, saturate_mask); + p->src_rgbl = p->linear->linearize(maskCopyTo(p->src, p->mask)); + p->dst->colors = maskCopyTo(p->dst->colors, p->mask); + p->dst_rgbl = p->dst->to(*(p->cs.l)).colors; + + // make no change for CCM_3x3, make change for CCM_4x3. + p->src_rgbl = p->prepare(p->src_rgbl); + + // distance function may affect the loss function and the fitting function + switch (p->distance) + { + case cv::ccm::DISTANCE_RGBL: + p->initialLeastSquare(true); + break; + default: + switch (p->initial_method_type) + { + case cv::ccm::INITIAL_METHOD_WHITE_BALANCE: + p->initialWhiteBalance(); + break; + case cv::ccm::INITIAL_METHOD_LEAST_SQUARE: + p->initialLeastSquare(); + break; + default: + CV_Error(Error::StsBadArg, "Wrong initial_methoddistance_type!" ); + break; + } + break; + } + p->fitting(); +} +Mat ColorCorrectionModel::getCCM() const +{ + return p->ccm; +} +double ColorCorrectionModel::getLoss() const +{ + return p->loss; +} +Mat ColorCorrectionModel::get_src_rgbl() const{ + return p->src_rgbl; +} +Mat ColorCorrectionModel::get_dst_rgbl() const{ + return p->dst_rgbl; +} +Mat ColorCorrectionModel::getMask() const{ + return p->mask; +} +Mat ColorCorrectionModel::getWeights() const{ + return p->weights; +} +} +} // namespace cv::ccm diff --git a/modules/mcc/src/color.cpp b/modules/mcc/src/color.cpp new file mode 100644 index 00000000000..ce00ac47e70 --- /dev/null +++ b/modules/mcc/src/color.cpp @@ -0,0 +1,398 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "color.hpp" + +namespace cv { +namespace ccm { +Color::Color() + : colors(Mat()) + , cs(*std::make_shared()) +{} +Color::Color(Mat colors_, enum COLOR_SPACE cs_) + : colors(colors_) + , cs(*GetCS::getInstance().get_cs(cs_)) +{} + +Color::Color(Mat colors_, const ColorSpace& cs_, Mat colored_) + : colors(colors_) + , cs(cs_) + , colored(colored_) +{ + grays = ~colored; +} +Color::Color(Mat colors_, enum COLOR_SPACE cs_, Mat colored_) + : colors(colors_) + , cs(*GetCS::getInstance().get_cs(cs_)) + , colored(colored_) +{ + grays = ~colored; +} + +Color::Color(Mat colors_, const ColorSpace& cs_) + : colors(colors_) + , cs(cs_) +{} + +Color Color::to(const ColorSpace& other, CAM method, bool save) +{ + if (history.count(other) == 1) + { + return *history[other]; + } + if (cs.relate(other)) + { + return Color(cs.relation(other).run(colors), other); + } + Operations ops; + ops.add(cs.to).add(XYZ(cs.io).cam(other.io, method)).add(other.from); + std::shared_ptr color(new Color(ops.run(colors), other)); + if (save) + { + history[other] = color; + } + return *color; +} +Color Color::to(COLOR_SPACE other, CAM method, bool save) +{ + return to(*GetCS::getInstance().get_cs(other), method, save); +} + +Mat Color::channel(Mat m, int i) +{ + Mat dchannels[3]; + split(m, dchannels); + return dchannels[i]; +} + +Mat Color::toGray(IO io, CAM method, bool save) +{ + XYZ xyz = *XYZ::get(io); + return channel(this->to(xyz, method, save).colors, 1); +} + +Mat Color::toLuminant(IO io, CAM method, bool save) +{ + Lab lab = *Lab::get(io); + return channel(this->to(lab, method, save).colors, 0); +} + +Mat Color::diff(Color& other, DISTANCE_TYPE method) +{ + return diff(other, cs.io, method); +} + +Mat Color::diff(Color& other, IO io, DISTANCE_TYPE method) +{ + Lab lab = *Lab::get(io); + switch (method) + { + case cv::ccm::DISTANCE_CIE76: + case cv::ccm::DISTANCE_CIE94_GRAPHIC_ARTS: + case cv::ccm::DISTANCE_CIE94_TEXTILES: + case cv::ccm::DISTANCE_CIE2000: + case cv::ccm::DISTANCE_CMC_1TO1: + case cv::ccm::DISTANCE_CMC_2TO1: + return distance(to(lab).colors, other.to(lab).colors, method); + case cv::ccm::DISTANCE_RGB: + return distance(to(*cs.nl).colors, other.to(*cs.nl).colors, method); + case cv::ccm::DISTANCE_RGBL: + return distance(to(*cs.l).colors, other.to(*cs.l).colors, method); + default: + CV_Error(Error::StsBadArg, "Wrong method!" ); + break; + } +} + +void Color::getGray(double JDN) +{ + if (!grays.empty()) + { + return; + } + Mat lab = to(COLOR_SPACE_Lab_D65_2).colors; + Mat gray(colors.size(), colors.type()); + int fromto[] = { 0, 0, -1, 1, -1, 2 }; + mixChannels(&lab, 1, &gray, 1, fromto, 3); + Mat d = distance(lab, gray, DISTANCE_CIE2000); + this->grays = d < JDN; + this->colored = ~grays; +} + +Color Color::operator[](Mat mask) +{ + return Color(maskCopyTo(colors, mask), cs); +} + +Mat GetColor::getColorChecker(const double* checker, int row) +{ + Mat res(row, 1, CV_64FC3); + for (int i = 0; i < row; ++i) + { + res.at(i, 0) = Vec3d(checker[3 * i], checker[3 * i + 1], checker[3 * i + 2]); + } + return res; +} + +Mat GetColor::getColorCheckerMASK(const uchar* checker, int row) +{ + Mat res(row, 1, CV_8U); + for (int i = 0; i < row; ++i) + { + res.at(i, 0) = checker[i]; + } + return res; +} + +std::shared_ptr GetColor::getColor(CONST_COLOR const_color) +{ + + /** @brief Data is from https://www.imatest.com/wp-content/uploads/2011/11/Lab-data-Iluminate-D65-D50-spectro.xls + see Miscellaneous.md for details. +*/ + static const double ColorChecker2005_LAB_D50_2[24][3] = { { 37.986, 13.555, 14.059 }, + { 65.711, 18.13, 17.81 }, + { 49.927, -4.88, -21.925 }, + { 43.139, -13.095, 21.905 }, + { 55.112, 8.844, -25.399 }, + { 70.719, -33.397, -0.199 }, + { 62.661, 36.067, 57.096 }, + { 40.02, 10.41, -45.964 }, + { 51.124, 48.239, 16.248 }, + { 30.325, 22.976, -21.587 }, + { 72.532, -23.709, 57.255 }, + { 71.941, 19.363, 67.857 }, + { 28.778, 14.179, -50.297 }, + { 55.261, -38.342, 31.37 }, + { 42.101, 53.378, 28.19 }, + { 81.733, 4.039, 79.819 }, + { 51.935, 49.986, -14.574 }, + { 51.038, -28.631, -28.638 }, + { 96.539, -0.425, 1.186 }, + { 81.257, -0.638, -0.335 }, + { 66.766, -0.734, -0.504 }, + { 50.867, -0.153, -0.27 }, + { 35.656, -0.421, -1.231 }, + { 20.461, -0.079, -0.973 } }; + + static const uchar ColorChecker2005_COLORED_MASK[24] = { 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0 }; + static const double Vinyl_LAB_D50_2[18][3] = { { 100, 0.00520000001, -0.0104 }, + { 73.0833969, -0.819999993, -2.02099991 }, + { 62.493, 0.425999999, -2.23099995 }, + { 50.4640007, 0.446999997, -2.32399988 }, + { 37.7970009, 0.0359999985, -1.29700005 }, + { 0, 0, 0 }, + { 51.5880013, 73.5179977, 51.5690002 }, + { 93.6989975, -15.7340002, 91.9420013 }, + { 69.4079971, -46.5940018, 50.4869995 }, + { 66.61000060000001, -13.6789999, -43.1720009 }, + { 11.7110004, 16.9799995, -37.1759987 }, + { 51.973999, 81.9440002, -8.40699959 }, + { 40.5489998, 50.4399986, 24.8490009 }, + { 60.8160019, 26.0690002, 49.4420013 }, + { 52.2529984, -19.9500008, -23.9960003 }, + { 51.2859993, 48.4700012, -15.0579996 }, + { 68.70700069999999, 12.2959995, 16.2129993 }, + { 63.6839981, 10.2930002, 16.7639999 } }; + static const uchar Vinyl_COLORED_MASK[18] = { 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1 }; + static const double DigitalSG_LAB_D50_2[140][3] = { { 96.55, -0.91, 0.57 }, + { 6.43, -0.06, -0.41 }, + { 49.7, -0.18, 0.03 }, + { 96.5, -0.89, 0.59 }, + { 6.5, -0.06, -0.44 }, + { 49.66, -0.2, 0.01 }, + { 96.52, -0.91, 0.58 }, + { 6.49, -0.02, -0.28 }, + { 49.72, -0.2, 0.04 }, + { 96.43, -0.91, 0.67 }, + { 49.72, -0.19, 0 }, + { 32.6, 51.58, -10.85 }, + { 60.75, 26.22, -18.6 }, + { 28.69, 48.28, -39 }, + { 49.38, -15.43, -48.48 }, + { 60.63, -30.77, -26.23 }, + { 19.29, -26.37, -6.15 }, + { 60.15, -41.77, -12.6 }, + { 21.42, 1.67, 8.79 }, + { 49.69, -0.2, 0.01 }, + { 6.5, -0.03, -0.67 }, + { 21.82, 17.33, -18.35 }, + { 41.53, 18.48, -37.26 }, + { 19.99, -0.16, -36.29 }, + { 60.16, -18.45, -31.42 }, + { 19.94, -17.92, -20.96 }, + { 60.68, -6.05, -32.81 }, + { 50.81, -49.8, -9.63 }, + { 60.65, -39.77, 20.76 }, + { 6.53, -0.03, -0.43 }, + { 96.56, -0.91, 0.59 }, + { 84.19, -1.95, -8.23 }, + { 84.75, 14.55, 0.23 }, + { 84.87, -19.07, -0.82 }, + { 85.15, 13.48, 6.82 }, + { 84.17, -10.45, 26.78 }, + { 61.74, 31.06, 36.42 }, + { 64.37, 20.82, 18.92 }, + { 50.4, -53.22, 14.62 }, + { 96.51, -0.89, 0.65 }, + { 49.74, -0.19, 0.03 }, + { 31.91, 18.62, 21.99 }, + { 60.74, 38.66, 70.97 }, + { 19.35, 22.23, -58.86 }, + { 96.52, -0.91, 0.62 }, + { 6.66, 0, -0.3 }, + { 76.51, 20.81, 22.72 }, + { 72.79, 29.15, 24.18 }, + { 22.33, -20.7, 5.75 }, + { 49.7, -0.19, 0.01 }, + { 6.53, -0.05, -0.61 }, + { 63.42, 20.19, 19.22 }, + { 34.94, 11.64, -50.7 }, + { 52.03, -44.15, 39.04 }, + { 79.43, 0.29, -0.17 }, + { 30.67, -0.14, -0.53 }, + { 63.6, 14.44, 26.07 }, + { 64.37, 14.5, 17.05 }, + { 60.01, -44.33, 8.49 }, + { 6.63, -0.01, -0.47 }, + { 96.56, -0.93, 0.59 }, + { 46.37, -5.09, -24.46 }, + { 47.08, 52.97, 20.49 }, + { 36.04, 64.92, 38.51 }, + { 65.05, 0, -0.32 }, + { 40.14, -0.19, -0.38 }, + { 43.77, 16.46, 27.12 }, + { 64.39, 17, 16.59 }, + { 60.79, -29.74, 41.5 }, + { 96.48, -0.89, 0.64 }, + { 49.75, -0.21, 0.01 }, + { 38.18, -16.99, 30.87 }, + { 21.31, 29.14, -27.51 }, + { 80.57, 3.85, 89.61 }, + { 49.71, -0.2, 0.03 }, + { 60.27, 0.08, -0.41 }, + { 67.34, 14.45, 16.9 }, + { 64.69, 16.95, 18.57 }, + { 51.12, -49.31, 44.41 }, + { 49.7, -0.2, 0.02 }, + { 6.67, -0.05, -0.64 }, + { 51.56, 9.16, -26.88 }, + { 70.83, -24.26, 64.77 }, + { 48.06, 55.33, -15.61 }, + { 35.26, -0.09, -0.24 }, + { 75.16, 0.25, -0.2 }, + { 44.54, 26.27, 38.93 }, + { 35.91, 16.59, 26.46 }, + { 61.49, -52.73, 47.3 }, + { 6.59, -0.05, -0.5 }, + { 96.58, -0.9, 0.61 }, + { 68.93, -34.58, -0.34 }, + { 69.65, 20.09, 78.57 }, + { 47.79, -33.18, -30.21 }, + { 15.94, -0.42, -1.2 }, + { 89.02, -0.36, -0.48 }, + { 63.43, 25.44, 26.25 }, + { 65.75, 22.06, 27.82 }, + { 61.47, 17.1, 50.72 }, + { 96.53, -0.89, 0.66 }, + { 49.79, -0.2, 0.03 }, + { 85.17, 10.89, 17.26 }, + { 89.74, -16.52, 6.19 }, + { 84.55, 5.07, -6.12 }, + { 84.02, -13.87, -8.72 }, + { 70.76, 0.07, -0.35 }, + { 45.59, -0.05, 0.23 }, + { 20.3, 0.07, -0.32 }, + { 61.79, -13.41, 55.42 }, + { 49.72, -0.19, 0.02 }, + { 6.77, -0.05, -0.44 }, + { 21.85, 34.37, 7.83 }, + { 42.66, 67.43, 48.42 }, + { 60.33, 36.56, 3.56 }, + { 61.22, 36.61, 17.32 }, + { 62.07, 52.8, 77.14 }, + { 72.42, -9.82, 89.66 }, + { 62.03, 3.53, 57.01 }, + { 71.95, -27.34, 73.69 }, + { 6.59, -0.04, -0.45 }, + { 49.77, -0.19, 0.04 }, + { 41.84, 62.05, 10.01 }, + { 19.78, 29.16, -7.85 }, + { 39.56, 65.98, 33.71 }, + { 52.39, 68.33, 47.84 }, + { 81.23, 24.12, 87.51 }, + { 81.8, 6.78, 95.75 }, + { 71.72, -16.23, 76.28 }, + { 20.31, 14.45, 16.74 }, + { 49.68, -0.19, 0.05 }, + { 96.48, -0.88, 0.68 }, + { 49.69, -0.18, 0.03 }, + { 6.39, -0.04, -0.33 }, + { 96.54, -0.9, 0.67 }, + { 49.72, -0.18, 0.05 }, + { 6.49, -0.03, -0.41 }, + { 96.51, -0.9, 0.69 }, + { 49.7, -0.19, 0.07 }, + { 6.47, 0, -0.38 }, + { 96.46, -0.89, 0.7 } }; + + switch (const_color) + { + + case cv::ccm::COLORCHECKER_Macbeth: + { + Mat ColorChecker2005_LAB_D50_2_ = GetColor::getColorChecker(*ColorChecker2005_LAB_D50_2, 24); + Mat ColorChecker2005_COLORED_MASK_ = GetColor::getColorCheckerMASK(ColorChecker2005_COLORED_MASK, 24); + std::shared_ptr Macbeth_D50_2 = std::make_shared(ColorChecker2005_LAB_D50_2_, COLOR_SPACE_Lab_D50_2, ColorChecker2005_COLORED_MASK_); + return Macbeth_D50_2; + } + + case cv::ccm::COLORCHECKER_Vinyl: + { + Mat Vinyl_LAB_D50_2__ = GetColor::getColorChecker(*Vinyl_LAB_D50_2, 18); + Mat Vinyl_COLORED_MASK__ = GetColor::getColorCheckerMASK(Vinyl_COLORED_MASK, 18); + std::shared_ptr Vinyl_D50_2 = std::make_shared(Vinyl_LAB_D50_2__, COLOR_SPACE_Lab_D50_2, Vinyl_COLORED_MASK__); + return Vinyl_D50_2; + } + + case cv::ccm::COLORCHECKER_DigitalSG: + { + Mat DigitalSG_LAB_D50_2__ = GetColor::getColorChecker(*DigitalSG_LAB_D50_2, 140); + std::shared_ptr DigitalSG_D50_2 = std::make_shared(DigitalSG_LAB_D50_2__, COLOR_SPACE_Lab_D50_2); + return DigitalSG_D50_2; + } + } + CV_Error(Error::StsNotImplemented, ""); +} + +} +} // namespace cv::ccm diff --git a/modules/mcc/src/color.hpp b/modules/mcc/src/color.hpp new file mode 100644 index 00000000000..57ead3558c7 --- /dev/null +++ b/modules/mcc/src/color.hpp @@ -0,0 +1,118 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_COLOR_HPP__ +#define __OPENCV_MCC_COLOR_HPP__ + +#include "distance.hpp" +#include "colorspace.hpp" +#include "opencv2/mcc/ccm.hpp" + +namespace cv { +namespace ccm { + +/** @brief Color defined by color_values and color space +*/ + +class Color +{ +public: + /** @param grays mask of grayscale color + @param colored mask of colored color + @param history storage of historical conversion + */ + Mat colors; + const ColorSpace& cs; + Mat grays; + Mat colored; + std::map> history; + Color(); + Color(Mat colors_, enum COLOR_SPACE cs); + Color(Mat colors_, enum COLOR_SPACE cs, Mat colored); + Color(Mat colors_, const ColorSpace& cs, Mat colored); + Color(Mat colors_, const ColorSpace& cs); + virtual ~Color() {}; + + /** @brief Change to other color space. + The conversion process incorporates linear transformations to speed up. + @param other type of ColorSpace. + @param method the chromatic adapation method. + @param save when save if True, get data from history first. + @return Color. + */ + Color to(COLOR_SPACE other, CAM method = BRADFORD, bool save = true); + Color to(const ColorSpace& other, CAM method = BRADFORD, bool save = true); + + /** @brief Channels split. + @return each channel. + */ + Mat channel(Mat m, int i); + + /** @brief To Gray. + */ + Mat toGray(IO io, CAM method = BRADFORD, bool save = true); + + /** @brief To Luminant. + */ + Mat toLuminant(IO io, CAM method = BRADFORD, bool save = true); + + /** @brief Diff without IO. + @param other type of Color. + @param method type of distance. + @return distance between self and other + */ + Mat diff(Color& other, DISTANCE_TYPE method = DISTANCE_CIE2000); + + /** @brief Diff with IO. + @param other type of Color. + @param io type of IO. + @param method type of distance. + @return distance between self and other + */ + Mat diff(Color& other, IO io, DISTANCE_TYPE method = DISTANCE_CIE2000); + + /** @brief Calculate gray mask. + */ + void getGray(double JDN = 2.0); + + /** @brief Operator for mask copy. + */ + Color operator[](Mat mask); +}; + +class GetColor +{ +public: + static std::shared_ptr getColor(CONST_COLOR const_color); + static Mat getColorChecker(const double* checker, int row); + static Mat getColorCheckerMASK(const uchar* checker, int row); +}; + +} +} // namespace cv::ccm + +#endif \ No newline at end of file diff --git a/modules/mcc/src/colorspace.cpp b/modules/mcc/src/colorspace.cpp new file mode 100644 index 00000000000..9dfe3f6e125 --- /dev/null +++ b/modules/mcc/src/colorspace.cpp @@ -0,0 +1,789 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "precomp.hpp" + +#include "colorspace.hpp" +#include "operations.hpp" +#include "io.hpp" + +namespace cv { +namespace ccm { +static const std::vector& getIlluminants(const IO& io) +{ + static const std::map> illuminants = { + { IO::getIOs(A_2), { 1.098466069456375, 1, 0.3558228003436005 } }, + { IO::getIOs(A_10), { 1.111420406956693, 1, 0.3519978321919493 } }, + { IO::getIOs(D50_2), { 0.9642119944211994, 1, 0.8251882845188288 } }, + { IO::getIOs(D50_10), { 0.9672062750333777, 1, 0.8142801513128616 } }, + { IO::getIOs(D55_2), { 0.956797052643698, 1, 0.9214805860173273 } }, + { IO::getIOs(D55_10), { 0.9579665682254781, 1, 0.9092525159847462 } }, + { IO::getIOs(D65_2), { 0.95047, 1., 1.08883 } }, + { IO::getIOs(D65_10), { 0.94811, 1., 1.07304 } }, + { IO::getIOs(D75_2), { 0.9497220898840717, 1, 1.226393520724154 } }, + { IO::getIOs(D75_10), { 0.9441713925645873, 1, 1.2064272211720228 } }, + { IO::getIOs(E_2), { 1., 1., 1. } }, + { IO::getIOs(E_10), { 1., 1., 1. } }, + }; + auto it = illuminants.find(io); + CV_Assert(it != illuminants.end()); + return it->second; +}; + +/* @brief Basic class for ColorSpace. + */ +bool ColorSpace::relate(const ColorSpace& other) const +{ + return (type == other.type) && (io == other.io); +}; + +Operations ColorSpace::relation(const ColorSpace& /*other*/) const +{ + return Operations::get_IDENTITY_OPS(); +} + +bool ColorSpace::operator<(const ColorSpace& other) const +{ + return (io < other.io || (io == other.io && type < other.type) || (io == other.io && type == other.type && linear < other.linear)); +} + +/* @brief Base of RGB color space; + * the argument values are from AdobeRGB; + * Data from https://en.wikipedia.org/wiki/Adobe_RGB_color_space + */ +Operations RGBBase_::relation(const ColorSpace& other) const +{ + if (linear == other.linear) + { + return Operations::get_IDENTITY_OPS(); + } + if (linear) + { + return Operations({ Operation(fromL) }); + } + return Operations({ Operation(toL) }); +} + +/* @brief Initial operations. + */ +void RGBBase_::init() +{ + setParameter(); + calLinear(); + calM(); + calOperations(); +} + +/* @brief Produce color space instance with linear and non-linear versions. + * @param rgbl type of RGBBase_. + */ +void RGBBase_::bind(RGBBase_& rgbl) +{ + init(); + rgbl.init(); + l = &rgbl; + rgbl.l = &rgbl; + nl = this; + rgbl.nl = this; +} + +/* @brief Calculation of M_RGBL2XYZ_base. + */ +void RGBBase_::calM() +{ + Mat XYZr, XYZg, XYZb, XYZ_rgbl, Srgb; + XYZr = Mat(xyY2XYZ({ xr, yr }), true); + XYZg = Mat(xyY2XYZ({ xg, yg }), true); + XYZb = Mat(xyY2XYZ({ xb, yb }), true); + merge(std::vector { XYZr, XYZg, XYZb }, XYZ_rgbl); + XYZ_rgbl = XYZ_rgbl.reshape(1, XYZ_rgbl.rows); + Mat XYZw = Mat(getIlluminants(io), true); + solve(XYZ_rgbl, XYZw, Srgb); + merge(std::vector { Srgb.at(0) * XYZr, Srgb.at(1) * XYZg, + Srgb.at(2) * XYZb }, + M_to); + M_to = M_to.reshape(1, M_to.rows); + M_from = M_to.inv(); +}; + +/* @brief operations to or from XYZ. + */ +void RGBBase_::calOperations() +{ + // rgb -> rgbl + toL = [this](Mat rgb) -> Mat { return toLFunc(rgb); }; + + // rgbl -> rgb + fromL = [this](Mat rgbl) -> Mat { return fromLFunc(rgbl); }; + + if (linear) + { + to = Operations({ Operation(M_to.t()) }); + from = Operations({ Operation(M_from.t()) }); + } + else + { + to = Operations({ Operation(toL), Operation(M_to.t()) }); + from = Operations({ Operation(M_from.t()), Operation(fromL) }); + } +} + +Mat RGBBase_::toLFunc(Mat& /*rgb*/) { return Mat(); } + +Mat RGBBase_::fromLFunc(Mat& /*rgbl*/) { return Mat(); } + +/* @brief Base of Adobe RGB color space; + */ + +Mat AdobeRGBBase_::toLFunc(Mat& rgb) { return gammaCorrection(rgb, gamma); } + +Mat AdobeRGBBase_::fromLFunc(Mat& rgbl) +{ + return gammaCorrection(rgbl, 1. / gamma); +} + +/* @brief Base of sRGB color space; + */ + +void sRGBBase_::calLinear() +{ + alpha = a + 1; + K0 = a / (gamma - 1); + phi = (pow(alpha, gamma) * pow(gamma - 1, gamma - 1)) / (pow(a, gamma - 1) * pow(gamma, gamma)); + beta = K0 / phi; +} + +/* @brief Used by toLFunc. + */ +double sRGBBase_::toLFuncEW(double& x) +{ + if (x > K0) + { + return pow(((x + alpha - 1) / alpha), gamma); + } + else if (x >= -K0) + { + return x / phi; + } + else + { + return -(pow(((-x + alpha - 1) / alpha), gamma)); + } +} + +/* @brief Linearization. + * @param rgb the input array, type of cv::Mat. + * @return the output array, type of cv::Mat. + */ +Mat sRGBBase_::toLFunc(Mat& rgb) +{ + return elementWise(rgb, + [this](double a_) -> double { return toLFuncEW(a_); }); +} + +/* @brief Used by fromLFunc. + */ +double sRGBBase_::fromLFuncEW(double& x) +{ + if (x > beta) + { + return alpha * pow(x, 1 / gamma) - (alpha - 1); + } + else if (x >= -beta) + { + return x * phi; + } + else + { + return -(alpha * pow(-x, 1 / gamma) - (alpha - 1)); + } +} + +/* @brief Delinearization. + * @param rgbl the input array, type of cv::Mat. + * @return the output array, type of cv::Mat. + */ +Mat sRGBBase_::fromLFunc(Mat& rgbl) +{ + return elementWise(rgbl, + [this](double a_) -> double { return fromLFuncEW(a_); }); +} + +/* @brief sRGB color space. + * data from https://en.wikipedia.org/wiki/SRGB. + */ +void sRGB_::setParameter() +{ + xr = 0.64; + yr = 0.33; + xg = 0.3; + yg = 0.6; + xb = 0.15; + yb = 0.06; + a = 0.055; + gamma = 2.4; +} + +/* @brief Adobe RGB color space. + */ +void AdobeRGB_::setParameter() +{ + xr = 0.64; + yr = 0.33; + xg = 0.21; + yg = 0.71; + xb = 0.15; + yb = 0.06; + gamma = 2.2; +} + +/* @brief Wide-gamut RGB color space. + * data from https://en.wikipedia.org/wiki/Wide-gamut_RGB_color_space. + */ +void WideGamutRGB_::setParameter() +{ + xr = 0.7347; + yr = 0.2653; + xg = 0.1152; + yg = 0.8264; + xb = 0.1566; + yb = 0.0177; + gamma = 2.2; +} + +/* @brief ProPhoto RGB color space. + * data from https://en.wikipedia.org/wiki/ProPhoto_RGB_color_space. + */ +void ProPhotoRGB_::setParameter() +{ + xr = 0.734699; + yr = 0.265301; + xg = 0.159597; + yg = 0.840403; + xb = 0.036598; + yb = 0.000105; + gamma = 1.8; +} + +/* @brief DCI-P3 RGB color space. + * data from https://en.wikipedia.org/wiki/DCI-P3. + */ + +void DCI_P3_RGB_::setParameter() +{ + xr = 0.68; + yr = 0.32; + xg = 0.265; + yg = 0.69; + xb = 0.15; + yb = 0.06; + gamma = 2.2; +} + +/* @brief Apple RGB color space. + * data from + * http://www.brucelindbloom.com/index.html?WorkingSpaceInfo.html. + */ +void AppleRGB_::setParameter() +{ + xr = 0.625; + yr = 0.34; + xg = 0.28; + yg = 0.595; + xb = 0.155; + yb = 0.07; + gamma = 1.8; +} + +/* @brief REC_709 RGB color space. + * data from https://en.wikipedia.org/wiki/Rec._709. + */ +void REC_709_RGB_::setParameter() +{ + xr = 0.64; + yr = 0.33; + xg = 0.3; + yg = 0.6; + xb = 0.15; + yb = 0.06; + a = 0.099; + gamma = 1 / 0.45; +} + +/* @brief REC_2020 RGB color space. + * data from https://en.wikipedia.org/wiki/Rec._2020. + */ + +void REC_2020_RGB_::setParameter() +{ + xr = 0.708; + yr = 0.292; + xg = 0.17; + yg = 0.797; + xb = 0.131; + yb = 0.046; + a = 0.09929682680944; + gamma = 1 / 0.45; +} + +Operations XYZ::cam(IO dio, CAM method) +{ + return (io == dio) ? Operations() + : Operations({ Operation(cam_(io, dio, method).t()) }); +} +Mat XYZ::cam_(IO sio, IO dio, CAM method) const +{ + static std::map, Mat> cams; + + if (sio == dio) + { + return Mat::eye(cv::Size(3, 3), CV_64FC1); + } + if (cams.count(std::make_tuple(dio, sio, method)) == 1) + { + return cams[std::make_tuple(dio, sio, method)]; + } + /* @brief XYZ color space. + * Chromatic adaption matrices. + */ + + static const Mat Von_Kries = (Mat_(3, 3) << 0.40024, 0.7076, -0.08081, -0.2263, 1.16532, 0.0457, 0., 0., 0.91822); + static const Mat Bradford = (Mat_(3, 3) << 0.8951, 0.2664, -0.1614, -0.7502, 1.7135, 0.0367, 0.0389, -0.0685, 1.0296); + static const std::map> MAs = { + { IDENTITY, { Mat::eye(Size(3, 3), CV_64FC1), Mat::eye(Size(3, 3), CV_64FC1) } }, + { VON_KRIES, { Von_Kries, Von_Kries.inv() } }, + { BRADFORD, { Bradford, Bradford.inv() } } + }; + + // Function from http://www.brucelindbloom.com/index.html?ColorCheckerRGB.html. + Mat XYZws = Mat(getIlluminants(dio)); + Mat XYZWd = Mat(getIlluminants(sio)); + Mat MA = MAs.at(method)[0]; + Mat MA_inv = MAs.at(method)[1]; + Mat M = MA_inv * Mat::diag((MA * XYZws) / (MA * XYZWd)) * MA; + cams[std::make_tuple(dio, sio, method)] = M; + cams[std::make_tuple(sio, dio, method)] = M.inv(); + return M; +} + +std::shared_ptr XYZ::get(IO io) +{ + static std::map> xyz_cs; + + if (xyz_cs.count(io) == 1) + { + return xyz_cs[io]; + } + std::shared_ptr XYZ_CS = std::make_shared(io); + xyz_cs[io] = XYZ_CS; + return xyz_cs[io]; +} + +/* @brief Lab color space. + */ +Lab::Lab(IO io_) + : ColorSpace(io_, "Lab", true) +{ + to = { Operation([this](Mat src) -> Mat { return tosrc(src); }) }; + from = { Operation([this](Mat src) -> Mat { return fromsrc(src); }) }; +} + +Vec3d Lab::fromxyz(cv::Vec3d& xyz) +{ + auto& il = getIlluminants(io); + double x = xyz[0] / il[0], + y = xyz[1] / il[1], + z = xyz[2] / il[2]; + auto f = [](double t) -> double { + return t > t0 ? std::cbrt(t) : (m * t + c); + }; + double fx = f(x), fy = f(y), fz = f(z); + return { 116. * fy - 16., 500 * (fx - fy), 200 * (fy - fz) }; +} + +/* @brief Calculate From. + * @param src the input array, type of cv::Mat. + * @return the output array, type of cv::Mat + */ +Mat Lab::fromsrc(Mat& src) +{ + return channelWise(src, + [this](cv::Vec3d a) -> cv::Vec3d { return fromxyz(a); }); +} + +Vec3d Lab::tolab(cv::Vec3d& lab) +{ + auto f_inv = [](double t) -> double { + return t > delta ? pow(t, 3.0) : (t - c) / m; + }; + double L = (lab[0] + 16.) / 116., a = lab[1] / 500., b = lab[2] / 200.; + auto& il = getIlluminants(io); + return { il[0] * f_inv(L + a), + il[1] * f_inv(L), + il[2] * f_inv(L - b) }; +} + +/* @brief Calculate To. + * @param src the input array, type of cv::Mat. + * @return the output array, type of cv::Mat + */ +Mat Lab::tosrc(Mat& src) +{ + return channelWise(src, + [this](cv::Vec3d a) -> cv::Vec3d { return tolab(a); }); +} + +std::shared_ptr Lab::get(IO io) +{ + static std::map> lab_cs; + + if (lab_cs.count(io) == 1) + { + return lab_cs[io]; + } + std::shared_ptr Lab_CS(new Lab(io)); + lab_cs[io] = Lab_CS; + return lab_cs[io]; +} + +GetCS::GetCS() +{ + // nothing +} + +GetCS& GetCS::getInstance() +{ + static GetCS instance; + return instance; +} + +std::shared_ptr GetCS::get_rgb(enum COLOR_SPACE cs_name) +{ + switch (cs_name) + { + case cv::ccm::COLOR_SPACE_sRGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr sRGB_CS(new sRGB_(false)); + std::shared_ptr sRGBL_CS(new sRGB_(true)); + (*sRGB_CS).bind(*sRGBL_CS); + map_cs[COLOR_SPACE_sRGB] = sRGB_CS; + map_cs[COLOR_SPACE_sRGBL] = sRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_AdobeRGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr AdobeRGB_CS(new AdobeRGB_(false)); + std::shared_ptr AdobeRGBL_CS(new AdobeRGB_(true)); + (*AdobeRGB_CS).bind(*AdobeRGBL_CS); + map_cs[COLOR_SPACE_AdobeRGB] = AdobeRGB_CS; + map_cs[COLOR_SPACE_AdobeRGBL] = AdobeRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_WideGamutRGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr WideGamutRGB_CS(new WideGamutRGB_(false)); + std::shared_ptr WideGamutRGBL_CS(new WideGamutRGB_(true)); + (*WideGamutRGB_CS).bind(*WideGamutRGBL_CS); + map_cs[COLOR_SPACE_WideGamutRGB] = WideGamutRGB_CS; + map_cs[COLOR_SPACE_WideGamutRGBL] = WideGamutRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_ProPhotoRGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr ProPhotoRGB_CS(new ProPhotoRGB_(false)); + std::shared_ptr ProPhotoRGBL_CS(new ProPhotoRGB_(true)); + (*ProPhotoRGB_CS).bind(*ProPhotoRGBL_CS); + map_cs[COLOR_SPACE_ProPhotoRGB] = ProPhotoRGB_CS; + map_cs[COLOR_SPACE_ProPhotoRGBL] = ProPhotoRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_DCI_P3_RGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr DCI_P3_RGB_CS(new DCI_P3_RGB_(false)); + std::shared_ptr DCI_P3_RGBL_CS(new DCI_P3_RGB_(true)); + (*DCI_P3_RGB_CS).bind(*DCI_P3_RGBL_CS); + map_cs[COLOR_SPACE_DCI_P3_RGB] = DCI_P3_RGB_CS; + map_cs[COLOR_SPACE_DCI_P3_RGBL] = DCI_P3_RGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_AppleRGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr AppleRGB_CS(new AppleRGB_(false)); + std::shared_ptr AppleRGBL_CS(new AppleRGB_(true)); + (*AppleRGB_CS).bind(*AppleRGBL_CS); + map_cs[COLOR_SPACE_AppleRGB] = AppleRGB_CS; + map_cs[COLOR_SPACE_AppleRGBL] = AppleRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_REC_709_RGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr REC_709_RGB_CS(new REC_709_RGB_(false)); + std::shared_ptr REC_709_RGBL_CS(new REC_709_RGB_(true)); + (*REC_709_RGB_CS).bind(*REC_709_RGBL_CS); + map_cs[COLOR_SPACE_REC_709_RGB] = REC_709_RGB_CS; + map_cs[COLOR_SPACE_REC_709_RGBL] = REC_709_RGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_REC_2020_RGB: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr REC_2020_RGB_CS(new REC_2020_RGB_(false)); + std::shared_ptr REC_2020_RGBL_CS(new REC_2020_RGB_(true)); + (*REC_2020_RGB_CS).bind(*REC_2020_RGBL_CS); + map_cs[COLOR_SPACE_REC_2020_RGB] = REC_2020_RGB_CS; + map_cs[COLOR_SPACE_REC_2020_RGBL] = REC_2020_RGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_sRGBL: + case cv::ccm::COLOR_SPACE_AdobeRGBL: + case cv::ccm::COLOR_SPACE_WideGamutRGBL: + case cv::ccm::COLOR_SPACE_ProPhotoRGBL: + case cv::ccm::COLOR_SPACE_DCI_P3_RGBL: + case cv::ccm::COLOR_SPACE_AppleRGBL: + case cv::ccm::COLOR_SPACE_REC_709_RGBL: + case cv::ccm::COLOR_SPACE_REC_2020_RGBL: + CV_Error(Error::StsBadArg, "linear RGB colorspaces are not supported, you should assigned as normal RGB color space"); + break; + + default: + CV_Error(Error::StsBadArg, "Only RGB color spaces are supported"); + } + return (std::dynamic_pointer_cast)(map_cs[cs_name]); +} + +std::shared_ptr GetCS::get_cs(enum COLOR_SPACE cs_name) +{ + switch (cs_name) + { + case cv::ccm::COLOR_SPACE_sRGB: + case cv::ccm::COLOR_SPACE_sRGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr sRGB_CS(new sRGB_(false)); + std::shared_ptr sRGBL_CS(new sRGB_(true)); + (*sRGB_CS).bind(*sRGBL_CS); + map_cs[COLOR_SPACE_sRGB] = sRGB_CS; + map_cs[COLOR_SPACE_sRGBL] = sRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_AdobeRGB: + case cv::ccm::COLOR_SPACE_AdobeRGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr AdobeRGB_CS(new AdobeRGB_(false)); + std::shared_ptr AdobeRGBL_CS(new AdobeRGB_(true)); + (*AdobeRGB_CS).bind(*AdobeRGBL_CS); + map_cs[COLOR_SPACE_AdobeRGB] = AdobeRGB_CS; + map_cs[COLOR_SPACE_AdobeRGBL] = AdobeRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_WideGamutRGB: + case cv::ccm::COLOR_SPACE_WideGamutRGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr WideGamutRGB_CS(new WideGamutRGB_(false)); + std::shared_ptr WideGamutRGBL_CS(new WideGamutRGB_(true)); + (*WideGamutRGB_CS).bind(*WideGamutRGBL_CS); + map_cs[COLOR_SPACE_WideGamutRGB] = WideGamutRGB_CS; + map_cs[COLOR_SPACE_WideGamutRGBL] = WideGamutRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_ProPhotoRGB: + case cv::ccm::COLOR_SPACE_ProPhotoRGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr ProPhotoRGB_CS(new ProPhotoRGB_(false)); + std::shared_ptr ProPhotoRGBL_CS(new ProPhotoRGB_(true)); + (*ProPhotoRGB_CS).bind(*ProPhotoRGBL_CS); + map_cs[COLOR_SPACE_ProPhotoRGB] = ProPhotoRGB_CS; + map_cs[COLOR_SPACE_ProPhotoRGBL] = ProPhotoRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_DCI_P3_RGB: + case cv::ccm::COLOR_SPACE_DCI_P3_RGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr DCI_P3_RGB_CS(new DCI_P3_RGB_(false)); + std::shared_ptr DCI_P3_RGBL_CS(new DCI_P3_RGB_(true)); + (*DCI_P3_RGB_CS).bind(*DCI_P3_RGBL_CS); + map_cs[COLOR_SPACE_DCI_P3_RGB] = DCI_P3_RGB_CS; + map_cs[COLOR_SPACE_DCI_P3_RGBL] = DCI_P3_RGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_AppleRGB: + case cv::ccm::COLOR_SPACE_AppleRGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr AppleRGB_CS(new AppleRGB_(false)); + std::shared_ptr AppleRGBL_CS(new AppleRGB_(true)); + (*AppleRGB_CS).bind(*AppleRGBL_CS); + map_cs[COLOR_SPACE_AppleRGB] = AppleRGB_CS; + map_cs[COLOR_SPACE_AppleRGBL] = AppleRGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_REC_709_RGB: + case cv::ccm::COLOR_SPACE_REC_709_RGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr REC_709_RGB_CS(new REC_709_RGB_(false)); + std::shared_ptr REC_709_RGBL_CS(new REC_709_RGB_(true)); + (*REC_709_RGB_CS).bind(*REC_709_RGBL_CS); + map_cs[COLOR_SPACE_REC_709_RGB] = REC_709_RGB_CS; + map_cs[COLOR_SPACE_REC_709_RGBL] = REC_709_RGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_REC_2020_RGB: + case cv::ccm::COLOR_SPACE_REC_2020_RGBL: + { + if (map_cs.count(cs_name) < 1) + { + std::shared_ptr REC_2020_RGB_CS(new REC_2020_RGB_(false)); + std::shared_ptr REC_2020_RGBL_CS(new REC_2020_RGB_(true)); + (*REC_2020_RGB_CS).bind(*REC_2020_RGBL_CS); + map_cs[COLOR_SPACE_REC_2020_RGB] = REC_2020_RGB_CS; + map_cs[COLOR_SPACE_REC_2020_RGBL] = REC_2020_RGBL_CS; + } + break; + } + case cv::ccm::COLOR_SPACE_XYZ_D65_2: + return XYZ::get(IO::getIOs(D65_2)); + break; + case cv::ccm::COLOR_SPACE_XYZ_D50_2: + return XYZ::get(IO::getIOs(D50_2)); + break; + case cv::ccm::COLOR_SPACE_XYZ_D65_10: + return XYZ::get(IO::getIOs(D65_10)); + break; + case cv::ccm::COLOR_SPACE_XYZ_D50_10: + return XYZ::get(IO::getIOs(D50_10)); + break; + case cv::ccm::COLOR_SPACE_XYZ_A_2: + return XYZ::get(IO::getIOs(A_2)); + break; + case cv::ccm::COLOR_SPACE_XYZ_A_10: + return XYZ::get(IO::getIOs(A_10)); + break; + case cv::ccm::COLOR_SPACE_XYZ_D55_2: + return XYZ::get(IO::getIOs(D55_2)); + break; + case cv::ccm::COLOR_SPACE_XYZ_D55_10: + return XYZ::get(IO::getIOs(D55_10)); + break; + case cv::ccm::COLOR_SPACE_XYZ_D75_2: + return XYZ::get(IO::getIOs(D75_2)); + break; + case cv::ccm::COLOR_SPACE_XYZ_D75_10: + return XYZ::get(IO::getIOs(D75_10)); + break; + case cv::ccm::COLOR_SPACE_XYZ_E_2: + return XYZ::get(IO::getIOs(E_2)); + break; + case cv::ccm::COLOR_SPACE_XYZ_E_10: + return XYZ::get(IO::getIOs(E_10)); + break; + case cv::ccm::COLOR_SPACE_Lab_D65_2: + return Lab::get(IO::getIOs(D65_2)); + break; + case cv::ccm::COLOR_SPACE_Lab_D50_2: + return Lab::get(IO::getIOs(D50_2)); + break; + case cv::ccm::COLOR_SPACE_Lab_D65_10: + return Lab::get(IO::getIOs(D65_10)); + break; + case cv::ccm::COLOR_SPACE_Lab_D50_10: + return Lab::get(IO::getIOs(D50_10)); + break; + case cv::ccm::COLOR_SPACE_Lab_A_2: + return Lab::get(IO::getIOs(A_2)); + break; + case cv::ccm::COLOR_SPACE_Lab_A_10: + return Lab::get(IO::getIOs(A_10)); + break; + case cv::ccm::COLOR_SPACE_Lab_D55_2: + return Lab::get(IO::getIOs(D55_2)); + break; + case cv::ccm::COLOR_SPACE_Lab_D55_10: + return Lab::get(IO::getIOs(D55_10)); + break; + case cv::ccm::COLOR_SPACE_Lab_D75_2: + return Lab::get(IO::getIOs(D75_2)); + break; + case cv::ccm::COLOR_SPACE_Lab_D75_10: + return Lab::get(IO::getIOs(D75_10)); + break; + case cv::ccm::COLOR_SPACE_Lab_E_2: + return Lab::get(IO::getIOs(E_2)); + break; + case cv::ccm::COLOR_SPACE_Lab_E_10: + return Lab::get(IO::getIOs(E_10)); + break; + default: + break; + } + + return map_cs[cs_name]; +} + +} +} // namespace cv::ccm diff --git a/modules/mcc/src/colorspace.hpp b/modules/mcc/src/colorspace.hpp new file mode 100644 index 00000000000..57b5bc2ff40 --- /dev/null +++ b/modules/mcc/src/colorspace.hpp @@ -0,0 +1,365 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_COLORSPACE_HPP__ +#define __OPENCV_MCC_COLORSPACE_HPP__ + +#include "operations.hpp" +#include "io.hpp" +#include "opencv2/mcc/ccm.hpp" + +namespace cv { +namespace ccm { + +/** @brief Basic class for ColorSpace. +*/ +class ColorSpace +{ +public: + typedef std::function MatFunc; + IO io; + std::string type; + bool linear; + Operations to; + Operations from; + ColorSpace* l; + ColorSpace* nl; + + ColorSpace() {}; + + ColorSpace(IO io_, std::string type_, bool linear_) + : io(io_) + , type(type_) + , linear(linear_) {}; + + virtual ~ColorSpace() + { + l = 0; + nl = 0; + }; + virtual bool relate(const ColorSpace& other) const; + + virtual Operations relation(const ColorSpace& /*other*/) const; + + bool operator<(const ColorSpace& other) const; +}; + +/** @brief Base of RGB color space; + the argument values are from AdobeRGB; + Data from https://en.wikipedia.org/wiki/Adobe_RGB_color_space +*/ + +class RGBBase_ : public ColorSpace +{ +public: + // primaries + double xr; + double yr; + double xg; + double yg; + double xb; + double yb; + MatFunc toL; + MatFunc fromL; + Mat M_to; + Mat M_from; + + using ColorSpace::ColorSpace; + + /** @brief There are 3 kinds of relationships for RGB: + 1. Different types; - no operation + 1. Same type, same linear; - copy + 2. Same type, different linear, self is nonlinear; - 2 toL + 3. Same type, different linear, self is linear - 3 fromL + @param other type of ColorSpace. + @return Operations. + */ + Operations relation(const ColorSpace& other) const CV_OVERRIDE; + + /** @brief Initial operations. + */ + void init(); + /** @brief Produce color space instance with linear and non-linear versions. + @param rgbl type of RGBBase_. + */ + void bind(RGBBase_& rgbl); + +private: + virtual void setParameter() {}; + + /** @brief Calculation of M_RGBL2XYZ_base. + */ + virtual void calM(); + + /** @brief operations to or from XYZ. + */ + virtual void calOperations(); + + virtual void calLinear() {}; + + virtual Mat toLFunc(Mat& /*rgb*/); + + virtual Mat fromLFunc(Mat& /*rgbl*/); +}; + +/** @brief Base of Adobe RGB color space; +*/ +class AdobeRGBBase_ : public RGBBase_ + +{ +public: + using RGBBase_::RGBBase_; + double gamma; + +private: + Mat toLFunc(Mat& rgb) CV_OVERRIDE; + Mat fromLFunc(Mat& rgbl) CV_OVERRIDE; +}; + +/** @brief Base of sRGB color space; +*/ +class sRGBBase_ : public RGBBase_ + +{ +public: + using RGBBase_::RGBBase_; + double a; + double gamma; + double alpha; + double beta; + double phi; + double K0; + +private: + /** @brief linearization parameters + */ + virtual void calLinear() CV_OVERRIDE; + /** @brief Used by toLFunc. + */ + double toLFuncEW(double& x); + + /** @brief Linearization. + @param rgb the input array, type of cv::Mat. + @return the output array, type of cv::Mat. + */ + Mat toLFunc(Mat& rgb) CV_OVERRIDE; + + /** @brief Used by fromLFunc. + */ + double fromLFuncEW(double& x); + + /** @brief Delinearization. + @param rgbl the input array, type of cv::Mat. + @return the output array, type of cv::Mat. + */ + Mat fromLFunc(Mat& rgbl) CV_OVERRIDE; +}; + +/** @brief sRGB color space. + data from https://en.wikipedia.org/wiki/SRGB. +*/ +class sRGB_ : public sRGBBase_ + +{ +public: + sRGB_(bool linear_) + : sRGBBase_(IO::getIOs(D65_2), "sRGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief Adobe RGB color space. +*/ +class AdobeRGB_ : public AdobeRGBBase_ +{ +public: + AdobeRGB_(bool linear_ = false) + : AdobeRGBBase_(IO::getIOs(D65_2), "AdobeRGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief Wide-gamut RGB color space. + data from https://en.wikipedia.org/wiki/Wide-gamut_RGB_color_space. +*/ +class WideGamutRGB_ : public AdobeRGBBase_ +{ +public: + WideGamutRGB_(bool linear_ = false) + : AdobeRGBBase_(IO::getIOs(D50_2), "WideGamutRGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief ProPhoto RGB color space. + data from https://en.wikipedia.org/wiki/ProPhoto_RGB_color_space. +*/ + +class ProPhotoRGB_ : public AdobeRGBBase_ +{ +public: + ProPhotoRGB_(bool linear_ = false) + : AdobeRGBBase_(IO::getIOs(D50_2), "ProPhotoRGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief DCI-P3 RGB color space. + data from https://en.wikipedia.org/wiki/DCI-P3. +*/ +class DCI_P3_RGB_ : public AdobeRGBBase_ +{ +public: + DCI_P3_RGB_(bool linear_ = false) + : AdobeRGBBase_(IO::getIOs(D65_2), "DCI_P3_RGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief Apple RGB color space. + data from http://www.brucelindbloom.com/index.html?WorkingSpaceInfo.html. +*/ +class AppleRGB_ : public AdobeRGBBase_ +{ +public: + AppleRGB_(bool linear_ = false) + : AdobeRGBBase_(IO::getIOs(D65_2), "AppleRGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief REC_709 RGB color space. + data from https://en.wikipedia.org/wiki/Rec._709. +*/ +class REC_709_RGB_ : public sRGBBase_ +{ +public: + REC_709_RGB_(bool linear_) + : sRGBBase_(IO::getIOs(D65_2), "REC_709_RGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief REC_2020 RGB color space. + data from https://en.wikipedia.org/wiki/Rec._2020. +*/ +class REC_2020_RGB_ : public sRGBBase_ +{ +public: + REC_2020_RGB_(bool linear_) + : sRGBBase_(IO::getIOs(D65_2), "REC_2020_RGB", linear_) {}; + +private: + void setParameter() CV_OVERRIDE; +}; + +/** @brief Enum of the possible types of CAMs. +*/ +enum CAM +{ + IDENTITY, + VON_KRIES, + BRADFORD +}; + + +/** @brief XYZ color space. + Chromatic adaption matrices. +*/ +class XYZ : public ColorSpace +{ +public: + XYZ(IO io_) + : ColorSpace(io_, "XYZ", true) {}; + Operations cam(IO dio, CAM method = BRADFORD); + static std::shared_ptr get(IO io); + +private: + /** @brief Get cam. + @param sio the input IO of src. + @param dio the input IO of dst. + @param method type of CAM. + @return the output array, type of cv::Mat. + */ + Mat cam_(IO sio, IO dio, CAM method = BRADFORD) const; +}; + +/** @brief Lab color space. +*/ +class Lab : public ColorSpace +{ +public: + Lab(IO io); + static std::shared_ptr get(IO io); + +private: + static constexpr double delta = (6. / 29.); + static constexpr double m = 1. / (3. * delta * delta); + static constexpr double t0 = delta * delta * delta; + static constexpr double c = 4. / 29.; + + Vec3d fromxyz(Vec3d& xyz); + + /** @brief Calculate From. + @param src the input array, type of cv::Mat. + @return the output array, type of cv::Mat + */ + Mat fromsrc(Mat& src); + + Vec3d tolab(Vec3d& lab); + + /** @brief Calculate To. + @param src the input array, type of cv::Mat. + @return the output array, type of cv::Mat + */ + Mat tosrc(Mat& src); +}; + +class GetCS +{ +protected: + std::map> map_cs; + + GetCS(); // singleton, use getInstance() +public: + static GetCS& getInstance(); + + std::shared_ptr get_rgb(enum COLOR_SPACE cs_name); + std::shared_ptr get_cs(enum COLOR_SPACE cs_name); +}; + +} +} // namespace cv::ccm + +#endif \ No newline at end of file diff --git a/modules/mcc/src/distance.cpp b/modules/mcc/src/distance.cpp new file mode 100644 index 00000000000..7996379e939 --- /dev/null +++ b/modules/mcc/src/distance.cpp @@ -0,0 +1,222 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "distance.hpp" + +namespace cv { +namespace ccm { + +double deltaCIE76(const Vec3d& lab1, const Vec3d& lab2) { return norm(lab1 - lab2); }; + +double deltaCIE94(const Vec3d& lab1, const Vec3d& lab2, const double& kH, + const double& kC, const double& kL, const double& k1, const double& k2) +{ + double dl = lab1[0] - lab2[0]; + double c1 = sqrt(pow(lab1[1], 2) + pow(lab1[2], 2)); + double c2 = sqrt(pow(lab2[1], 2) + pow(lab2[2], 2)); + double dc = c1 - c2; + double da = lab1[1] - lab2[1]; + double db = lab1[2] - lab2[2]; + double dh = pow(da, 2) + pow(db, 2) - pow(dc, 2); + double sc = 1.0 + k1 * c1; + double sh = 1.0 + k2 * c1; + double sl = 1.0; + double res = pow(dl / (kL * sl), 2) + pow(dc / (kC * sc), 2) + dh / pow(kH * sh, 2); + + return res > 0 ? sqrt(res) : 0; +} + +double deltaCIE94GraphicArts(const Vec3d& lab1, const Vec3d& lab2) +{ + return deltaCIE94(lab1, lab2); +} + +double toRad(const double& degree) { return degree / 180 * CV_PI; }; + +double deltaCIE94Textiles(const Vec3d& lab1, const Vec3d& lab2) +{ + return deltaCIE94(lab1, lab2, 1.0, 1.0, 2.0, 0.048, 0.014); +} + +double deltaCIEDE2000_(const Vec3d& lab1, const Vec3d& lab2, const double& kL, + const double& kC, const double& kH) +{ + double delta_L_apo = lab2[0] - lab1[0]; + double l_bar_apo = (lab1[0] + lab2[0]) / 2.0; + double C1 = sqrt(pow(lab1[1], 2) + pow(lab1[2], 2)); + double C2 = sqrt(pow(lab2[1], 2) + pow(lab2[2], 2)); + double C_bar = (C1 + C2) / 2.0; + double G = sqrt(pow(C_bar, 7) / (pow(C_bar, 7) + pow(25, 7))); + double a1_apo = lab1[1] + lab1[1] / 2.0 * (1.0 - G); + double a2_apo = lab2[1] + lab2[1] / 2.0 * (1.0 - G); + double C1_apo = sqrt(pow(a1_apo, 2) + pow(lab1[2], 2)); + double C2_apo = sqrt(pow(a2_apo, 2) + pow(lab2[2], 2)); + double C_bar_apo = (C1_apo + C2_apo) / 2.0; + double delta_C_apo = C2_apo - C1_apo; + + double h1_apo; + if (C1_apo == 0) + { + h1_apo = 0.0; + } + else + { + h1_apo = atan2(lab1[2], a1_apo); + if (h1_apo < 0.0) + h1_apo += 2. * CV_PI; + } + + double h2_apo; + if (C2_apo == 0) + { + h2_apo = 0.0; + } + else + { + h2_apo = atan2(lab2[2], a2_apo); + if (h2_apo < 0.0) + h2_apo += 2. * CV_PI; + } + + double delta_h_apo; + if (abs(h2_apo - h1_apo) <= CV_PI) + { + delta_h_apo = h2_apo - h1_apo; + } + else if (h2_apo <= h1_apo) + { + delta_h_apo = h2_apo - h1_apo + 2. * CV_PI; + } + else + { + delta_h_apo = h2_apo - h1_apo - 2. * CV_PI; + } + + double H_bar_apo; + if (C1_apo == 0 || C2_apo == 0) + { + H_bar_apo = h1_apo + h2_apo; + } + else if (abs(h1_apo - h2_apo) <= CV_PI) + { + H_bar_apo = (h1_apo + h2_apo) / 2.0; + } + else if (h1_apo + h2_apo < 2. * CV_PI) + { + H_bar_apo = (h1_apo + h2_apo + 2. * CV_PI) / 2.0; + } + else + { + H_bar_apo = (h1_apo + h2_apo - 2. * CV_PI) / 2.0; + } + + double delta_H_apo = 2.0 * sqrt(C1_apo * C2_apo) * sin(delta_h_apo / 2.0); + double T = 1.0 - 0.17 * cos(H_bar_apo - toRad(30.)) + 0.24 * cos(2.0 * H_bar_apo) + 0.32 * cos(3.0 * H_bar_apo + toRad(6.0)) - 0.2 * cos(4.0 * H_bar_apo - toRad(63.0)); + double sC = 1.0 + 0.045 * C_bar_apo; + double sH = 1.0 + 0.015 * C_bar_apo * T; + double sL = 1.0 + ((0.015 * pow(l_bar_apo - 50.0, 2.0)) / sqrt(20.0 + pow(l_bar_apo - 50.0, 2.0))); + double RT = -2.0 * G * sin(toRad(60.0) * exp(-pow((H_bar_apo - toRad(275.0)) / toRad(25.0), 2.0))); + double res = (pow(delta_L_apo / (kL * sL), 2.0) + pow(delta_C_apo / (kC * sC), 2.0) + pow(delta_H_apo / (kH * sH), 2.0) + RT * (delta_C_apo / (kC * sC)) * (delta_H_apo / (kH * sH))); + return res > 0 ? sqrt(res) : 0; +} + +double deltaCIEDE2000(const Vec3d& lab1, const Vec3d& lab2) +{ + return deltaCIEDE2000_(lab1, lab2); +} + +double deltaCMC(const Vec3d& lab1, const Vec3d& lab2, const double& kL, const double& kC) +{ + double dL = lab2[0] - lab1[0]; + double da = lab2[1] - lab1[1]; + double db = lab2[2] - lab1[2]; + double C1 = sqrt(pow(lab1[1], 2.0) + pow(lab1[2], 2.0)); + double C2 = sqrt(pow(lab2[1], 2.0) + pow(lab2[2], 2.0)); + double dC = C2 - C1; + double dH = sqrt(pow(da, 2) + pow(db, 2) - pow(dC, 2)); + + double H1; + if (C1 == 0.) + { + H1 = 0.0; + } + else + { + H1 = atan2(lab1[2], lab1[1]); + if (H1 < 0.0) + H1 += 2. * CV_PI; + } + + double F = pow(C1, 2) / sqrt(pow(C1, 4) + 1900); + double T = (H1 > toRad(164) && H1 <= toRad(345)) + ? 0.56 + abs(0.2 * cos(H1 + toRad(168))) + : 0.36 + abs(0.4 * cos(H1 + toRad(35))); + double sL = lab1[0] < 16. ? 0.511 : (0.040975 * lab1[0]) / (1.0 + 0.01765 * lab1[0]); + double sC = (0.0638 * C1) / (1.0 + 0.0131 * C1) + 0.638; + double sH = sC * (F * T + 1.0 - F); + + return sqrt(pow(dL / (kL * sL), 2.0) + pow(dC / (kC * sC), 2.0) + pow(dH / sH, 2.0)); +} + +double deltaCMC1To1(const Vec3d& lab1, const Vec3d& lab2) +{ + return deltaCMC(lab1, lab2); +} + +double deltaCMC2To1(const Vec3d& lab1, const Vec3d& lab2) +{ + return deltaCMC(lab1, lab2, 2, 1); +} + +Mat distance(Mat src, Mat ref, DISTANCE_TYPE distance_type) +{ + switch (distance_type) + { + case cv::ccm::DISTANCE_CIE76: + return distanceWise(src, ref, deltaCIE76); + case cv::ccm::DISTANCE_CIE94_GRAPHIC_ARTS: + return distanceWise(src, ref, deltaCIE94GraphicArts); + case cv::ccm::DISTANCE_CIE94_TEXTILES: + return distanceWise(src, ref, deltaCIE94Textiles); + case cv::ccm::DISTANCE_CIE2000: + return distanceWise(src, ref, deltaCIEDE2000); + case cv::ccm::DISTANCE_CMC_1TO1: + return distanceWise(src, ref, deltaCMC1To1); + case cv::ccm::DISTANCE_CMC_2TO1: + return distanceWise(src, ref, deltaCMC2To1); + case cv::ccm::DISTANCE_RGB: + return distanceWise(src, ref, deltaCIE76); + case cv::ccm::DISTANCE_RGBL: + return distanceWise(src, ref, deltaCIE76); + default: + CV_Error(Error::StsBadArg, "Wrong distance_type!" ); + break; + } +}; + +} +} // namespace ccm \ No newline at end of file diff --git a/modules/mcc/src/distance.hpp b/modules/mcc/src/distance.hpp new file mode 100644 index 00000000000..5acfc93cdb6 --- /dev/null +++ b/modules/mcc/src/distance.hpp @@ -0,0 +1,99 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_DISTANCE_HPP__ +#define __OPENCV_MCC_DISTANCE_HPP__ + +#include "utils.hpp" +#include "opencv2/mcc/ccm.hpp" + +namespace cv { +namespace ccm { +/** possibale functions to calculate the distance between + colors.see https://en.wikipedia.org/wiki/Color_difference for details;*/ + +/** @brief distance between two points in formula CIE76 + @param lab1 a 3D vector + @param lab2 a 3D vector + @return distance between lab1 and lab2 +*/ + +double deltaCIE76(const Vec3d& lab1, const Vec3d& lab2); + +/** @brief distance between two points in formula CIE94 + @param lab1 a 3D vector + @param lab2 a 3D vector + @param kH Hue scale + @param kC Chroma scale + @param kL Lightness scale + @param k1 first scale parameter + @param k2 second scale parameter + @return distance between lab1 and lab2 +*/ + +double deltaCIE94(const Vec3d& lab1, const Vec3d& lab2, const double& kH = 1.0, + const double& kC = 1.0, const double& kL = 1.0, const double& k1 = 0.045, + const double& k2 = 0.015); + +double deltaCIE94GraphicArts(const Vec3d& lab1, const Vec3d& lab2); + +double toRad(const double& degree); + +double deltaCIE94Textiles(const Vec3d& lab1, const Vec3d& lab2); + +/** @brief distance between two points in formula CIE2000 + @param lab1 a 3D vector + @param lab2 a 3D vector + @param kL Lightness scale + @param kC Chroma scale + @param kH Hue scale + @return distance between lab1 and lab2 +*/ +double deltaCIEDE2000_(const Vec3d& lab1, const Vec3d& lab2, const double& kL = 1.0, + const double& kC = 1.0, const double& kH = 1.0); +double deltaCIEDE2000(const Vec3d& lab1, const Vec3d& lab2); + +/** @brief distance between two points in formula CMC + @param lab1 a 3D vector + @param lab2 a 3D vector + @param kL Lightness scale + @param kC Chroma scale + @return distance between lab1 and lab2 +*/ + +double deltaCMC(const Vec3d& lab1, const Vec3d& lab2, const double& kL = 1, const double& kC = 1); + +double deltaCMC1To1(const Vec3d& lab1, const Vec3d& lab2); + +double deltaCMC2To1(const Vec3d& lab1, const Vec3d& lab2); + +Mat distance(Mat src,Mat ref, DISTANCE_TYPE distance_type); + +} +} // namespace cv::ccm + +#endif \ No newline at end of file diff --git a/modules/mcc/src/io.cpp b/modules/mcc/src/io.cpp new file mode 100644 index 00000000000..c9c5c2026aa --- /dev/null +++ b/modules/mcc/src/io.cpp @@ -0,0 +1,133 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "io.hpp" +namespace cv { +namespace ccm { +IO::IO(std::string illuminant_, std::string observer_) + : illuminant(illuminant_) + , observer(observer_) {}; + +bool IO::operator<(const IO& other) const +{ + return (illuminant < other.illuminant || ((illuminant == other.illuminant) && (observer < other.observer))); +} + +bool IO::operator==(const IO& other) const +{ + return illuminant == other.illuminant && observer == other.observer; +}; + +IO IO::getIOs(IO_TYPE io) +{ + switch (io) + { + case cv::ccm::A_2: + { + IO A_2_IO("A", "2"); + return A_2_IO; + break; + } + case cv::ccm::A_10: + { + IO A_1O_IO("A", "10"); + return A_1O_IO; + break; + } + case cv::ccm::D50_2: + { + IO D50_2_IO("D50", "2"); + return D50_2_IO; + break; + } + case cv::ccm::D50_10: + { + IO D50_10_IO("D50", "10"); + return D50_10_IO; + break; + } + case cv::ccm::D55_2: + { + IO D55_2_IO("D55", "2"); + return D55_2_IO; + break; + } + case cv::ccm::D55_10: + { + IO D55_10_IO("D55", "10"); + return D55_10_IO; + break; + } + case cv::ccm::D65_2: + { + IO D65_2_IO("D65", "2"); + return D65_2_IO; + } + case cv::ccm::D65_10: + { + IO D65_10_IO("D65", "10"); + return D65_10_IO; + break; + } + case cv::ccm::D75_2: + { + IO D75_2_IO("D75", "2"); + return D75_2_IO; + break; + } + case cv::ccm::D75_10: + { + IO D75_10_IO("D75", "10"); + return D75_10_IO; + break; + } + case cv::ccm::E_2: + { + IO E_2_IO("E", "2"); + return E_2_IO; + break; + } + case cv::ccm::E_10: + { + IO E_10_IO("E", "10"); + return E_10_IO; + break; + } + default: + return IO(); + break; + } +} +// data from https://en.wikipedia.org/wiki/Standard_illuminant. +std::vector xyY2XYZ(const std::vector& xyY) +{ + double Y = xyY.size() >= 3 ? xyY[2] : 1; + return { Y * xyY[0] / xyY[1], Y, Y / xyY[1] * (1 - xyY[0] - xyY[1]) }; +} + +} +} // namespace cv::ccm diff --git a/modules/mcc/src/io.hpp b/modules/mcc/src/io.hpp new file mode 100644 index 00000000000..c79864e3c41 --- /dev/null +++ b/modules/mcc/src/io.hpp @@ -0,0 +1,72 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_IO_HPP__ +#define __OPENCV_MCC_IO_HPP__ + +#include +#include + +namespace cv { +namespace ccm { + +enum IO_TYPE +{ + A_2, + A_10, + D50_2, + D50_10, + D55_2, + D55_10, + D65_2, + D65_10, + D75_2, + D75_10, + E_2, + E_10 +}; + +/** @brief Io is the meaning of illuminant and observer. See notes of ccm.hpp + for supported list for illuminant and observer*/ +class IO +{ +public: + std::string illuminant; + std::string observer; + IO() {}; + IO(std::string illuminant, std::string observer); + virtual ~IO() {}; + bool operator<(const IO& other) const; + bool operator==(const IO& other) const; + static IO getIOs(IO_TYPE io); +}; +std::vector xyY2XYZ(const std::vector& xyY); + +} +} // namespace cv::ccm + +#endif \ No newline at end of file diff --git a/modules/mcc/src/linearize.cpp b/modules/mcc/src/linearize.cpp new file mode 100644 index 00000000000..3a48a560eb8 --- /dev/null +++ b/modules/mcc/src/linearize.cpp @@ -0,0 +1,130 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "linearize.hpp" + +namespace cv { +namespace ccm { + +Polyfit::Polyfit(Mat x, Mat y, int deg_) + : deg(deg_) +{ + int n = x.cols * x.rows * x.channels(); + x = x.reshape(1, n); + y = y.reshape(1, n); + Mat_ A = Mat_::ones(n, deg + 1); + for (int i = 0; i < n; ++i) + { + for (int j = 1; j < A.cols; ++j) + { + A.at(i, j) = x.at(i) * A.at(i, j - 1); + } + } + Mat y_(y); + cv::solve(A, y_, p, DECOMP_SVD); +} + +Mat Polyfit::operator()(const Mat& inp) +{ + return elementWise(inp, [this](double x) -> double { return fromEW(x); }); +}; + +double Polyfit::fromEW(double x) +{ + double res = 0; + for (int d = 0; d <= deg; ++d) + { + res += pow(x, d) * p.at(d, 0); + } + return res; +}; + +LogPolyfit::LogPolyfit(Mat x, Mat y, int deg_) + : deg(deg_) +{ + Mat mask_ = (x > 0) & (y > 0); + Mat src_, dst_, s_, d_; + src_ = maskCopyTo(x, mask_); + dst_ = maskCopyTo(y, mask_); + log(src_, s_); + log(dst_, d_); + p = Polyfit(s_, d_, deg); +} + +Mat LogPolyfit::operator()(const Mat& inp) +{ + Mat mask_ = inp >= 0; + Mat y, y_, res; + log(inp, y); + y = p(y); + exp(y, y_); + y_.copyTo(res, mask_); + return res; +}; + +Mat Linear::linearize(Mat inp) +{ + return inp; +}; + +Mat LinearGamma::linearize(Mat inp) +{ + return gammaCorrection(inp, gamma); +}; + +std::shared_ptr getLinear(double gamma, int deg, Mat src, Color dst, Mat mask, RGBBase_ cs, LINEAR_TYPE linear_type) +{ + std::shared_ptr p = std::make_shared(); + switch (linear_type) + { + case cv::ccm::LINEARIZATION_IDENTITY: + p.reset(new LinearIdentity()); + break; + case cv::ccm::LINEARIZATION_GAMMA: + p.reset(new LinearGamma(gamma)); + break; + case cv::ccm::LINEARIZATION_COLORPOLYFIT: + p.reset(new LinearColor(deg, src, dst, mask, cs)); + break; + case cv::ccm::LINEARIZATION_COLORLOGPOLYFIT: + p.reset(new LinearColor(deg, src, dst, mask, cs)); + break; + case cv::ccm::LINEARIZATION_GRAYPOLYFIT: + p.reset(new LinearGray(deg, src, dst, mask, cs)); + break; + case cv::ccm::LINEARIZATION_GRAYLOGPOLYFIT: + p.reset(new LinearGray(deg, src, dst, mask, cs)); + break; + default: + CV_Error(Error::StsBadArg, "Wrong linear_type!" ); + break; + } + return p; +}; + +} +} // namespace cv::ccm \ No newline at end of file diff --git a/modules/mcc/src/linearize.hpp b/modules/mcc/src/linearize.hpp new file mode 100644 index 00000000000..a703b5c5293 --- /dev/null +++ b/modules/mcc/src/linearize.hpp @@ -0,0 +1,209 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_LINEARIZE_HPP__ +#define __OPENCV_MCC_LINEARIZE_HPP__ + +#include +#include +#include "color.hpp" +#include "opencv2/mcc/ccm.hpp" +namespace cv { +namespace ccm { + +/** @brief Polyfit model. +*/ +class Polyfit +{ +public: + int deg; + Mat p; + Polyfit() {}; + + /** @brief Polyfit method. + https://en.wikipedia.org/wiki/Polynomial_regression + polynomial: yi = a0 + a1*xi + a2*xi^2 + ... + an*xi^deg (i = 1,2,...,n) + and deduct: Ax = y + */ + Polyfit(Mat x, Mat y, int deg); + virtual ~Polyfit() {}; + Mat operator()(const Mat& inp); + +private: + double fromEW(double x); +}; + +/** @brief Logpolyfit model. +*/ +class LogPolyfit + +{ +public: + int deg; + Polyfit p; + + LogPolyfit() {}; + + /** @brief Logpolyfit method. + */ + LogPolyfit(Mat x, Mat y, int deg); + virtual ~LogPolyfit() {}; + Mat operator()(const Mat& inp); +}; + +/** @brief Linearization base. +*/ + +class Linear +{ +public: + Linear() {}; + virtual ~Linear() {}; + + /** @brief Inference. + @param inp the input array, type of cv::Mat. + */ + virtual Mat linearize(Mat inp); + /* *\brief Evaluate linearization model. + */ + virtual void value(void) {}; +}; + +/** @brief Linearization identity. + make no change. +*/ +class LinearIdentity : public Linear +{}; + +/** @brief Linearization gamma correction. +*/ +class LinearGamma : public Linear +{ +public: + double gamma; + + LinearGamma(double gamma_) + : gamma(gamma_) {}; + + Mat linearize(Mat inp) CV_OVERRIDE; +}; + +/** @brief Linearization. + Grayscale polynomial fitting. +*/ +template +class LinearGray : public Linear +{ +public: + int deg; + T p; + LinearGray(int deg_, Mat src, Color dst, Mat mask, RGBBase_ cs) + : deg(deg_) + { + dst.getGray(); + Mat lear_gray_mask = mask & dst.grays; + + // the grayscale function is approximate for src is in relative color space. + src = rgb2gray(maskCopyTo(src, lear_gray_mask)); + Mat dst_ = maskCopyTo(dst.toGray(cs.io), lear_gray_mask); + calc(src, dst_); + } + + /** @brief monotonically increase is not guaranteed. + @param src the input array, type of cv::Mat. + @param dst the input array, type of cv::Mat. + */ + void calc(const Mat& src, const Mat& dst) + { + p = T(src, dst, deg); + }; + + Mat linearize(Mat inp) CV_OVERRIDE + { + return p(inp); + }; +}; + +/** @brief Linearization. + Fitting channels respectively. +*/ +template +class LinearColor : public Linear +{ +public: + int deg; + T pr; + T pg; + T pb; + + LinearColor(int deg_, Mat src_, Color dst, Mat mask, RGBBase_ cs) + : deg(deg_) + { + Mat src = maskCopyTo(src_, mask); + Mat dst_ = maskCopyTo(dst.to(*cs.l).colors, mask); + calc(src, dst_); + } + + void calc(const Mat& src, const Mat& dst) + { + Mat schannels[3]; + Mat dchannels[3]; + split(src, schannels); + split(dst, dchannels); + pr = T(schannels[0], dchannels[0], deg); + pg = T(schannels[1], dchannels[1], deg); + pb = T(schannels[2], dchannels[2], deg); + }; + + Mat linearize(Mat inp) CV_OVERRIDE + { + Mat channels[3]; + split(inp, channels); + std::vector channel; + Mat res; + merge(std::vector { pr(channels[0]), pg(channels[1]), pb(channels[2]) }, res); + return res; + }; +}; + +/** @brief Get linearization method. + used in ccm model. + @param gamma used in LinearGamma. + @param deg degrees. + @param src the input array, type of cv::Mat. + @param dst the input array, type of cv::Mat. + @param mask the input array, type of cv::Mat. + @param cs type of RGBBase_. + @param linear_type type of linear. +*/ + +std::shared_ptr getLinear(double gamma, int deg, Mat src, Color dst, Mat mask, RGBBase_ cs, LINEAR_TYPE linear_type); + +} +} // namespace cv::ccm + +#endif diff --git a/modules/mcc/src/operations.cpp b/modules/mcc/src/operations.cpp new file mode 100644 index 00000000000..e4e76a2a270 --- /dev/null +++ b/modules/mcc/src/operations.cpp @@ -0,0 +1,90 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "operations.hpp" +#include "utils.hpp" +namespace cv { +namespace ccm { + +Mat Operation::operator()(Mat& abc) +{ + if (!linear) + { + return f(abc); + } + if (M.empty()) + { + return abc; + } + return multiple(abc, M); +}; + +void Operation::add(const Operation& other) +{ + if (M.empty()) + { + M = other.M.clone(); + } + else + { + M = M * other.M; + } +}; + +void Operation::clear() +{ + M = Mat(); +}; + +Operations& Operations::add(const Operations& other) +{ + ops.insert(ops.end(), other.ops.begin(), other.ops.end()); + return *this; +}; + +Mat Operations::run(Mat abc) +{ + Operation hd; + for (auto& op : ops) + { + if (op.linear) + { + hd.add(op); + } + else + { + abc = hd(abc); + hd.clear(); + abc = op(abc); + } + } + abc = hd(abc); + return abc; +}; + +} +} // namespace cv::ccm \ No newline at end of file diff --git a/modules/mcc/src/operations.hpp b/modules/mcc/src/operations.hpp new file mode 100644 index 00000000000..ae3b39b6019 --- /dev/null +++ b/modules/mcc/src/operations.hpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_OPERATIONS_HPP__ +#define __OPENCV_MCC_OPERATIONS_HPP__ + +#include "utils.hpp" + +namespace cv { +namespace ccm { + +/** @brief Operation class contains some operarions used for color space + conversion containing linear transformation and non-linear transformation + */ +class Operation +{ +public: + typedef std::function MatFunc; + bool linear; + Mat M; + MatFunc f; + + Operation() + : linear(true) + , M(Mat()) {}; + Operation(Mat M_) + : linear(true) + , M(M_) {}; + Operation(MatFunc f_) + : linear(false) + , f(f_) {}; + virtual ~Operation() {}; + + /** @brief operator function will run operation + */ + Mat operator()(Mat& abc); + + /** @brief add function will conbine this operation + with other linear transformation operation + */ + void add(const Operation& other); + + void clear(); + static Operation& get_IDENTITY_OP() + { + static Operation identity_op([](Mat x) { return x; }); + return identity_op; + } +}; + +class Operations +{ +public: + std::vector ops; + Operations() + : ops {} {}; + Operations(std::initializer_list op) + : ops { op } {}; + virtual ~Operations() {}; + + /** @brief add function will conbine this operation with other transformation operations + */ + Operations& add(const Operations& other); + + /** @brief run operations to make color conversion + */ + Mat run(Mat abc); + static const Operations& get_IDENTITY_OPS() + { + static Operations Operation_op {Operation::get_IDENTITY_OP()}; + return Operation_op; + } +}; + +} +} // namespace cv::ccm + +#endif \ No newline at end of file diff --git a/modules/mcc/src/utils.cpp b/modules/mcc/src/utils.cpp new file mode 100644 index 00000000000..3a0128b6ef6 --- /dev/null +++ b/modules/mcc/src/utils.cpp @@ -0,0 +1,119 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#include "utils.hpp" + +namespace cv { +namespace ccm { + +double gammaCorrection_(const double& element, const double& gamma) +{ + return (element >= 0 ? pow(element, gamma) : -pow((-element), gamma)); +} + +Mat gammaCorrection(const Mat& src, const double& gamma) +{ + return elementWise(src, [gamma](double element) -> double { return gammaCorrection_(element, gamma); }); +} + +Mat maskCopyTo(const Mat& src, const Mat& mask) +{ + Mat dst(countNonZero(mask), 1, src.type()); + const int channel = src.channels(); + auto it_mask = mask.begin(); + switch (channel) + { + case 1: + { + auto it_src = src.begin(), end_src = src.end(); + auto it_dst = dst.begin(); + for (; it_src != end_src; ++it_src, ++it_mask) + { + if (*it_mask) + { + (*it_dst) = (*it_src); + ++it_dst; + } + } + break; + } + case 3: + { + auto it_src = src.begin(), end_src = src.end(); + auto it_dst = dst.begin(); + for (; it_src != end_src; ++it_src, ++it_mask) + { + if (*it_mask) + { + (*it_dst) = (*it_src); + ++it_dst; + } + } + break; + } + default: + CV_Error(Error::StsBadArg, "Wrong channel!" ); + break; + } + return dst; +} + +Mat multiple(const Mat& xyz, const Mat& ccm) +{ + Mat tmp = xyz.reshape(1, xyz.rows * xyz.cols); + Mat res = tmp * ccm; + res = res.reshape(res.cols, xyz.rows); + return res; +} + +Mat saturate(Mat& src, const double& low, const double& up) +{ + Mat dst = Mat::ones(src.size(), CV_8UC1); + MatIterator_ it_src = src.begin(), end_src = src.end(); + MatIterator_ it_dst = dst.begin(); + for (; it_src != end_src; ++it_src, ++it_dst) + { + for (int i = 0; i < 3; ++i) + { + if ((*it_src)[i] > up || (*it_src)[i] < low) + { + *it_dst = 0; + break; + } + } + } + return dst; +} + +Mat rgb2gray(const Mat& rgb) +{ + const Matx31d m_gray(0.2126, 0.7152, 0.0722); + return multiple(rgb, Mat(m_gray)); +} + +} +} // namespace cv::ccm diff --git a/modules/mcc/src/utils.hpp b/modules/mcc/src/utils.hpp new file mode 100644 index 00000000000..02570ca0184 --- /dev/null +++ b/modules/mcc/src/utils.hpp @@ -0,0 +1,155 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright(C) 2020, Huawei Technologies Co.,Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Longbu Wang +// Jinheng Zhang +// Chenqi Shan + +#ifndef __OPENCV_MCC_UTILS_HPP__ +#define __OPENCV_MCC_UTILS_HPP__ + +#include + +namespace cv { +namespace ccm { + +double gammaCorrection_(const double& element, const double& gamma); + +/** @brief gamma correction. + \f[ + C_l=C_n^{\gamma},\qquad C_n\ge0\\ + C_l=-(-C_n)^{\gamma},\qquad C_n<0\\\\ + \f] + @param src the input array,type of Mat. + @param gamma a constant for gamma correction. + */ +Mat gammaCorrection(const Mat& src, const double& gamma); + +/** @brief maskCopyTo a function to delete unsatisfied elementwise. + @param src the input array, type of Mat. + @param mask operation mask that used to choose satisfided elementwise. + */ +Mat maskCopyTo(const Mat& src, const Mat& mask); + +/** @brief multiple the function used to compute an array with n channels + mulipied by ccm. + @param xyz the input array, type of Mat. + @param ccm the ccm matrix to make color correction. + */ +Mat multiple(const Mat& xyz, const Mat& ccm); + +/** @brief multiple the function used to get the mask of saturated colors, + colors between low and up will be choosed. + @param src the input array, type of Mat. + @param low the threshold to choose saturated colors + @param up the threshold to choose saturated colors +*/ +Mat saturate(Mat& src, const double& low, const double& up); + +/** @brief rgb2gray it is an approximation grayscale function for relative RGB + color space + @param rgb the input array,type of Mat. + */ +Mat rgb2gray(const Mat& rgb); + +/** @brief function for elementWise operation + @param src the input array, type of Mat + @param lambda a for operation + */ +template +Mat elementWise(const Mat& src, F&& lambda) +{ + Mat dst = src.clone(); + const int channel = src.channels(); + switch (channel) + { + case 1: + { + + MatIterator_ it, end; + for (it = dst.begin(), end = dst.end(); it != end; ++it) + { + (*it) = lambda((*it)); + } + break; + } + case 3: + { + MatIterator_ it, end; + for (it = dst.begin(), end = dst.end(); it != end; ++it) + { + for (int j = 0; j < 3; j++) + { + (*it)[j] = lambda((*it)[j]); + } + } + break; + } + default: + CV_Error(Error::StsBadArg, "Wrong channel!" ); + break; + } + return dst; +} + +/** @brief function for channel operation + @param src the input array, type of Mat + @param lambda the function for operation +*/ +template +Mat channelWise(const Mat& src, F&& lambda) +{ + Mat dst = src.clone(); + MatIterator_ it, end; + for (it = dst.begin(), end = dst.end(); it != end; ++it) + { + *it = lambda(*it); + } + return dst; +} + +/** @brief function for distance operation. + @param src the input array, type of Mat. + @param ref another input array, type of Mat. + @param lambda the computing method for distance . + */ +template +Mat distanceWise(Mat& src, Mat& ref, F&& lambda) +{ + Mat dst = Mat(src.size(), CV_64FC1); + MatIterator_ it_src = src.begin(), end_src = src.end(), + it_ref = ref.begin(); + MatIterator_ it_dst = dst.begin(); + for (; it_src != end_src; ++it_src, ++it_ref, ++it_dst) + { + *it_dst = lambda(*it_src, *it_ref); + } + return dst; +} + +Mat multiple(const Mat& xyz, const Mat& ccm); + +} +} // namespace cv::ccm + +#endif \ No newline at end of file diff --git a/modules/mcc/test/test_ccm.cpp b/modules/mcc/test/test_ccm.cpp new file mode 100644 index 00000000000..56ac51db410 --- /dev/null +++ b/modules/mcc/test/test_ccm.cpp @@ -0,0 +1,166 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test +{ +namespace +{ + +Mat s = (Mat_(24, 1) << + Vec3d(214.11, 98.67, 37.97), + Vec3d(231.94, 153.1, 85.27), + Vec3d(204.08, 143.71, 78.46), + Vec3d(190.58, 122.99, 30.84), + Vec3d(230.93, 148.46, 100.84), + Vec3d(228.64, 206.97, 97.5), + Vec3d(229.09, 137.07, 55.29), + Vec3d(189.21, 111.22, 92.66), + Vec3d(223.5, 96.42, 75.45), + Vec3d(201.82, 69.71, 50.9), + Vec3d(240.52, 196.47, 59.3), + Vec3d(235.73, 172.13, 54.), + Vec3d(131.6, 75.04, 68.86), + Vec3d(189.04, 170.43, 42.05), + Vec3d(222.23, 74., 71.95), + Vec3d(241.01, 199.1, 61.15), + Vec3d(224.99, 101.4, 100.24), + Vec3d(174.58, 152.63, 91.52), + Vec3d(248.06, 227.69, 140.5), + Vec3d(241.15, 201.38, 115.58), + Vec3d(236.49, 175.87, 88.86), + Vec3d(212.19, 133.49, 54.79), + Vec3d(181.17, 102.94, 36.18), + Vec3d(115.1, 53.77, 15.23)); + +TEST(CV_ccmRunColorCorrection, test_model) +{ + + ColorCorrectionModel model(s / 255, COLORCHECKER_Macbeth); + model.run(); + Mat src_rgbl = (Mat_(24, 1) << + Vec3d(0.68078957, 0.12382801, 0.01514889), + Vec3d(0.81177942, 0.32550452, 0.089818), + Vec3d(0.61259378, 0.2831933, 0.07478902), + Vec3d(0.52696493, 0.20105976, 0.00958657), + Vec3d(0.80402284, 0.30419523, 0.12989841), + Vec3d(0.78658646, 0.63184111, 0.12062068), + Vec3d(0.78999637, 0.25520249, 0.03462853), + Vec3d(0.51866697, 0.16114393, 0.1078387), + Vec3d(0.74820768, 0.11770076, 0.06862177), + Vec3d(0.59776825, 0.05765816, 0.02886627), + Vec3d(0.8793145, 0.56346033, 0.0403954), + Vec3d(0.84124847, 0.42120746, 0.03287592), + Vec3d(0.23333214, 0.06780408, 0.05612276), + Vec3d(0.5176423, 0.41210976, 0.01896255), + Vec3d(0.73888613, 0.06575388, 0.06181293), + Vec3d(0.88326036, 0.58018751, 0.04321991), + Vec3d(0.75922531, 0.13149072, 0.1282041), + Vec3d(0.4345097, 0.32331019, 0.10494139), + Vec3d(0.94110142, 0.77941419, 0.26946323), + Vec3d(0.88438952, 0.5949049 , 0.17536928), + Vec3d(0.84722687, 0.44160449, 0.09834799), + Vec3d(0.66743106, 0.24076803, 0.03394333), + Vec3d(0.47141286, 0.13592419, 0.01362205), + Vec3d(0.17377101, 0.03256864, 0.00203026)); + ASSERT_MAT_NEAR(src_rgbl, model.get_src_rgbl(), 1e-4); + + Mat dst_rgbl = (Mat_(24, 1) << + Vec3d(0.17303173, 0.08211037, 0.05672686), + Vec3d(0.56832031, 0.29269488, 0.21835529), + Vec3d(0.10365019, 0.19588357, 0.33140475), + Vec3d(0.10159676, 0.14892193, 0.05188294), + Vec3d(0.22159627, 0.21584476, 0.43461196), + Vec3d(0.10806379, 0.51437196, 0.41264213), + Vec3d(0.74736423, 0.20062878, 0.02807988), + Vec3d(0.05757947, 0.10516793, 0.40296109), + Vec3d(0.56676218, 0.08424805, 0.11969461), + Vec3d(0.11099515, 0.04230796, 0.14292554), + Vec3d(0.34546869, 0.50872001, 0.04944204), + Vec3d(0.79461323, 0.35942459, 0.02051968), + Vec3d(0.01710416, 0.05022043, 0.29220674), + Vec3d(0.05598012, 0.30021149, 0.06871162), + Vec3d(0.45585457, 0.03033727, 0.04085654), + Vec3d(0.85737614, 0.56757335, 0.0068503), + Vec3d(0.53348585, 0.08861148, 0.30750446), + Vec3d(-0.0374061, 0.24699498, 0.40041217), + Vec3d(0.91262695, 0.91493909, 0.89367049), + Vec3d(0.57981916, 0.59200418, 0.59328881), + Vec3d(0.35490581, 0.36544831, 0.36755375), + Vec3d(0.19007357, 0.19186587, 0.19308397), + Vec3d(0.08529188, 0.08887994, 0.09257601), + Vec3d(0.0303193, 0.03113818, 0.03274845)); + ASSERT_MAT_NEAR(dst_rgbl, model.get_dst_rgbl(), 1e-4); + + Mat mask = Mat::ones(24, 1, CV_8U); + ASSERT_MAT_NEAR(model.getMask(), mask, 0.0); + + + Mat ccm = (Mat_(3, 3) << + 0.37408717, 0.02066172, 0.05796725, + 0.12684056, 0.77364991, -0.01566532, + -0.27464866, 0.00652140, 2.74593262); + ASSERT_MAT_NEAR(model.getCCM(), ccm, 1e-4); +} +TEST(CV_ccmRunColorCorrection, test_masks_weights_1) +{ + Mat weights_list_ = (Mat_(24, 1) << + 1.1, 0, 0, 1.2, 0, 0, + 1.3, 0, 0, 1.4, 0, 0, + 0.5, 0, 0, 0.6, 0, 0, + 0.7, 0, 0, 0.8, 0, 0); + ColorCorrectionModel model1(s / 255,COLORCHECKER_Macbeth); + model1.setColorSpace(COLOR_SPACE_sRGB); + model1.setCCM_TYPE(CCM_3x3); + model1.setDistance(DISTANCE_CIE2000); + model1.setLinear(LINEARIZATION_GAMMA); + model1.setLinearGamma(2.2); + model1.setLinearDegree(3); + model1.setSaturatedThreshold(0, 0.98); + model1.setWeightsList(weights_list_); + model1.setWeightCoeff(1.5); + model1.run(); + Mat weights = (Mat_(8, 1) << + 1.15789474, 1.26315789, 1.36842105, 1.47368421, + 0.52631579, 0.63157895, 0.73684211, 0.84210526); + ASSERT_MAT_NEAR(model1.getWeights(), weights, 1e-4); + + Mat mask = (Mat_(24, 1) << + true, false, false, true, false, false, + true, false, false, true, false, false, + true, false, false, true, false, false, + true, false, false, true, false, false); + ASSERT_MAT_NEAR(model1.getMask(), mask, 0.0); +} + +TEST(CV_ccmRunColorCorrection, test_masks_weights_2) +{ + ColorCorrectionModel model2(s / 255, COLORCHECKER_Macbeth); + model2.setCCM_TYPE(CCM_3x3); + model2.setDistance(DISTANCE_CIE2000); + model2.setLinear(LINEARIZATION_GAMMA); + model2.setLinearGamma(2.2); + model2.setLinearDegree(3); + model2.setSaturatedThreshold(0.05, 0.93); + model2.setWeightsList(Mat()); + model2.setWeightCoeff(1.5); + model2.run(); + Mat weights = (Mat_(20, 1) << + 0.65554256, 1.49454705, 1.00499244, 0.79735434, 1.16327759, + 1.68623868, 1.37973155, 0.73213388, 1.0169629, 0.47430246, + 1.70312161, 0.45414218, 1.15910007, 0.7540434, 1.05049802, + 1.04551645, 1.54082353, 1.02453421, 0.6015915, 0.26154558); + ASSERT_MAT_NEAR(model2.getWeights(), weights, 1e-4); + + Mat mask = (Mat_(24, 1) << + true, true, true, true, true, true, + true, true, true, true, false, true, + true, true, true, false, true, true, + false, false, true, true, true, true); + ASSERT_MAT_NEAR(model2.getMask(), mask, 0.0); +} + +} // namespace +} // namespace opencv_test diff --git a/modules/mcc/test/test_precomp.hpp b/modules/mcc/test/test_precomp.hpp index 1f399441353..c4d81a348c5 100644 --- a/modules/mcc/test/test_precomp.hpp +++ b/modules/mcc/test/test_precomp.hpp @@ -6,11 +6,14 @@ #define __OPENCV_TEST_PRECOMP_HPP__ #include "opencv2/ts.hpp" +#include "opencv2/ts/cuda_test.hpp" #include "opencv2/mcc.hpp" +#include "opencv2/mcc/ccm.hpp" namespace opencv_test { using namespace cv::mcc; +using namespace cv::ccm; } #endif diff --git a/modules/mcc/tutorials/basic_ccm/color_correction_model.markdown b/modules/mcc/tutorials/basic_ccm/color_correction_model.markdown new file mode 100644 index 00000000000..76b98cd3f4b --- /dev/null +++ b/modules/mcc/tutorials/basic_ccm/color_correction_model.markdown @@ -0,0 +1,122 @@ +Color Correction Model{#tutorial_ccm_color_correction_model} +=========================== + +In this tutorial you will learn how to use the 'Color Correction Model' to do a color correction in a image. + +Reference +---- + +See details of ColorCorrection Algorithm at https://github.com/riskiest/color_calibration/tree/v4/doc/pdf/English/Algorithm + +Building +---- + +When building OpenCV, run the following command to build all the contrib modules: + +```make +cmake -D OPENCV_EXTRA_MODULES_PATH=/modules/ +``` + +Or only build the mcc module: + +```make +cmake -D OPENCV_EXTRA_MODULES_PATH=/modules/mcc +``` + +Or make sure you check the mcc module in the GUI version of CMake: cmake-gui. + +Source Code of the sample +----------- + +The sample has two parts of code, the first is the color checker detector model, see details at @ref tutorial_mcc_basic_chart_detection, the second part is to make collor calibration. + +``` +Here are the parameters for ColorCorrectionModel + src : + detected colors of ColorChecker patches; + NOTICE: the color type is RGB not BGR, and the color values are in [0, 1]; + constcolor : + the Built-in color card; + Supported list: + Macbeth: Macbeth ColorChecker ; + Vinyl: DKK ColorChecker ; + DigitalSG: DigitalSG ColorChecker with 140 squares; + Mat colors : + the reference color values + and corresponding color space + NOTICE: the color values are in [0, 1] + ref_cs : + the corresponding color space + If the color type is some RGB, the format is RGB not BGR; + Supported Color Space: + Supported list of RGB color spaces: + COLOR_SPACE_sRGB; + COLOR_SPACE_AdobeRGB; + COLOR_SPACE_WideGamutRGB; + COLOR_SPACE_ProPhotoRGB; + COLOR_SPACE_DCI_P3_RGB; + COLOR_SPACE_AppleRGB; + COLOR_SPACE_REC_709_RGB; + COLOR_SPACE_REC_2020_RGB; + Supported list of linear RGB color spaces: + COLOR_SPACE_sRGBL; + COLOR_SPACE_AdobeRGBL; + COLOR_SPACE_WideGamutRGBL; + COLOR_SPACE_ProPhotoRGBL; + COLOR_SPACE_DCI_P3_RGBL; + COLOR_SPACE_AppleRGBL; + COLOR_SPACE_REC_709_RGBL; + COLOR_SPACE_REC_2020_RGBL; + Supported list of non-RGB color spaces: + COLOR_SPACE_Lab_D50_2; + COLOR_SPACE_Lab_D65_2; + COLOR_SPACE_XYZ_D50_2; + COLOR_SPACE_XYZ_D65_2; + COLOR_SPACE_XYZ_D65_10; + COLOR_SPACE_XYZ_D50_10; + COLOR_SPACE_XYZ_A_2; + COLOR_SPACE_XYZ_A_10; + COLOR_SPACE_XYZ_D55_2; + COLOR_SPACE_XYZ_D55_10; + COLOR_SPACE_XYZ_D75_2; + COLOR_SPACE_XYZ_D75_10; + COLOR_SPACE_XYZ_E_2; + COLOR_SPACE_XYZ_E_10; + COLOR_SPACE_Lab_D65_10; + COLOR_SPACE_Lab_D50_10; + COLOR_SPACE_Lab_A_2; + COLOR_SPACE_Lab_A_10; + COLOR_SPACE_Lab_D55_2; + COLOR_SPACE_Lab_D55_10; + COLOR_SPACE_Lab_D75_2; + COLOR_SPACE_Lab_D75_10; + COLOR_SPACE_Lab_E_2; + COLOR_SPACE_Lab_E_10; +``` + + +## Code + +@snippet samples/color_correction_model.cpp tutorial + +## Explanation + +The first part is to detect the ColorChecker position. +@snippet samples/color_correction_model.cpp get_color_checker +@snippet samples/color_correction_model.cpp get_messages_of_image +Preparation for ColorChecker detection to get messages for the image. + +@snippet samples/color_correction_model.cpp create +The CCheckerDetectorobject is created and uses getListColorChecker function to get ColorChecker message. + +@snippet samples/color_correction_model.cpp get_ccm_Matrix +For every ColorChecker, we can compute a ccm matrix for color correction. Model1 is an object of ColorCorrectionModel class. The parameters should be changed to get the best effect of color correction. See other parameters' detail at the Parameters. + +@snippet samples/color_correction_model.cpp reference_color_values +If you use a customized ColorChecker, you can use your own reference color values and corresponding color space as shown above. + +@snippet samples/color_correction_model.cpp make_color_correction +The member function infer_image is to make correction correction using ccm matrix. + +@snippet samples/color_correction_model.cpp Save_calibrated_image +Save the calibrated image. diff --git a/modules/mcc/tutorials/table_of_content_ccm.markdown b/modules/mcc/tutorials/table_of_content_ccm.markdown new file mode 100644 index 00000000000..178f309b993 --- /dev/null +++ b/modules/mcc/tutorials/table_of_content_ccm.markdown @@ -0,0 +1,8 @@ +Color Correction Model {#tutorial_table_of_content_ccm} +=========================== + +- @subpage tutorial_ccm_color_correction_model + + *Author:* riskiest, shanchenqi, JinhengZhang + + How to do color correction, using Color Correction Model. \ No newline at end of file diff --git a/modules/optflow/src/rlof/berlof_invoker.hpp b/modules/optflow/src/rlof/berlof_invoker.hpp index 36e94f73644..ac866fe5344 100644 --- a/modules/optflow/src/rlof/berlof_invoker.hpp +++ b/modules/optflow/src/rlof/berlof_invoker.hpp @@ -187,7 +187,6 @@ class TrackerInvoker : public cv::ParallelLoopBody cv::Size _winSize = winSize; float MEstimatorScale = 1; int buffIdx = 0; - float c[8]; cv::Mat GMc0, GMc1, GMc2, GMc3; cv::Vec2f Mc0, Mc1, Mc2, Mc3; int noIteration = 0; @@ -572,6 +571,7 @@ class TrackerInvoker : public cv::ParallelLoopBody Mc3[0] = _b0[3]; Mc3[1] = _b1[3]; + float c[8] = {}; c[0] = -Mc0[0]; c[1] = -Mc1[0]; c[2] = -Mc2[0]; @@ -830,7 +830,6 @@ class TrackerInvoker : public cv::ParallelLoopBody int j; float MEstimatorScale = 1; int buffIdx = 0; - float c[8]; cv::Mat GMc0, GMc1, GMc2, GMc3; cv::Vec4f Mc0, Mc1, Mc2, Mc3; int noIteration = 0; @@ -1355,6 +1354,7 @@ class TrackerInvoker : public cv::ParallelLoopBody Mc3[3] = -_b3[3]; // + float c[8] = {}; c[0] = -Mc0[0]; c[1] = -Mc1[0]; c[2] = -Mc2[0]; @@ -1620,7 +1620,6 @@ class TrackerInvoker : public cv::ParallelLoopBody nextPt += halfWin; Point2f prevDelta(0,0); - float c[8]; for( j = 0; j < criteria.maxCount; j++ ) { cv::Point2f delta; @@ -1629,6 +1628,7 @@ class TrackerInvoker : public cv::ParallelLoopBody b = nextPt.y - cvFloor(nextPt.y); float ab = a * b; + float c[8] = {}; if( (inextPt.x != cvFloor(nextPt.x) || inextPt.y != cvFloor(nextPt.y) || j == 0)) { @@ -1996,7 +1996,6 @@ namespace radial { cv::Point2f backUpGain = gainVec; cv::Size _winSize = winSize; int j; - float c[8]; cv::Mat GMc0, GMc1, GMc2, GMc3; cv::Vec4f Mc0, Mc1, Mc2, Mc3; int noIteration = 0; @@ -2359,6 +2358,7 @@ namespace radial { Mc3[3] = -_b3[3]; // + float c[8] = {}; c[0] = -Mc0[0]; c[1] = -Mc1[0]; c[2] = -Mc2[0]; diff --git a/modules/phase_unwrapping/include/opencv2/phase_unwrapping/phase_unwrapping.hpp b/modules/phase_unwrapping/include/opencv2/phase_unwrapping/phase_unwrapping.hpp index 5b5cb5100ba..1091ff38029 100644 --- a/modules/phase_unwrapping/include/opencv2/phase_unwrapping/phase_unwrapping.hpp +++ b/modules/phase_unwrapping/include/opencv2/phase_unwrapping/phase_unwrapping.hpp @@ -58,9 +58,9 @@ class CV_EXPORTS_W PhaseUnwrapping : public virtual Algorithm /** * @brief Unwraps a 2D phase map. - * @param wrappedPhaseMap The wrapped phase map that needs to be unwrapped. + * @param wrappedPhaseMap The wrapped phase map of type CV_32FC1 that needs to be unwrapped. * @param unwrappedPhaseMap The unwrapped phase map. - * @param shadowMask Optional parameter used when some pixels do not hold any phase information in the wrapped phase map. + * @param shadowMask Optional CV_8UC1 mask image used when some pixels do not hold any phase information in the wrapped phase map. */ CV_WRAP virtual void unwrapPhaseMap( InputArray wrappedPhaseMap, OutputArray unwrappedPhaseMap, diff --git a/modules/phase_unwrapping/src/histogramphaseunwrapping.cpp b/modules/phase_unwrapping/src/histogramphaseunwrapping.cpp index 93e29674a03..82e202f04c7 100644 --- a/modules/phase_unwrapping/src/histogramphaseunwrapping.cpp +++ b/modules/phase_unwrapping/src/histogramphaseunwrapping.cpp @@ -402,6 +402,9 @@ void HistogramPhaseUnwrapping_Impl::unwrapPhaseMap( InputArray wrappedPhaseMap, temp.copyTo(mask); } + CV_CheckTypeEQ(wPhaseMap.type(), CV_32FC1, ""); + CV_CheckTypeEQ(mask.type(), CV_8UC1, ""); + computePixelsReliability(wPhaseMap, mask); computeEdgesReliabilityAndCreateHistogram(); diff --git a/modules/rgbd/CMakeLists.txt b/modules/rgbd/CMakeLists.txt index 7f2f6a67257..143cdf913af 100644 --- a/modules/rgbd/CMakeLists.txt +++ b/modules/rgbd/CMakeLists.txt @@ -1,2 +1,15 @@ set(the_description "RGBD algorithms") + +find_package(Ceres QUIET) ocv_define_module(rgbd opencv_core opencv_calib3d opencv_imgproc OPTIONAL opencv_viz WRAP python) + +if(Ceres_FOUND) + ocv_target_compile_definitions(${the_module} PUBLIC CERES_FOUND) + ocv_target_link_libraries(${the_module} ${CERES_LIBRARIES}) + if(Ceres_VERSION VERSION_LESS 2.0.0) + ocv_include_directories("${CERES_INCLUDE_DIRS}") + endif() + add_definitions(/DGLOG_NO_ABBREVIATED_SEVERITIES) # avoid ERROR macro conflict in glog (ceres dependency) +else() + message(STATUS "rgbd: CERES support is disabled. Ceres Solver is Required for Posegraph optimization") +endif() diff --git a/modules/rgbd/include/opencv2/rgbd.hpp b/modules/rgbd/include/opencv2/rgbd.hpp index 37b2927cdcf..d4ac749c2a5 100755 --- a/modules/rgbd/include/opencv2/rgbd.hpp +++ b/modules/rgbd/include/opencv2/rgbd.hpp @@ -13,6 +13,7 @@ #include "opencv2/rgbd/depth.hpp" #include "opencv2/rgbd/kinfu.hpp" #include "opencv2/rgbd/dynafu.hpp" +#include "opencv2/rgbd/large_kinfu.hpp" /** @defgroup rgbd RGB-Depth Processing diff --git a/modules/rgbd/include/opencv2/rgbd/depth.hpp b/modules/rgbd/include/opencv2/rgbd/depth.hpp index 94fdca62036..a50af0e1c6d 100755 --- a/modules/rgbd/include/opencv2/rgbd/depth.hpp +++ b/modules/rgbd/include/opencv2/rgbd/depth.hpp @@ -312,16 +312,17 @@ namespace rgbd depthTo3d(InputArray depth, InputArray K, OutputArray points3d, InputArray mask = noArray()); /** If the input image is of type CV_16UC1 (like the Kinect one), the image is converted to floats, divided - * by 1000 to get a depth in meters, and the values 0 are converted to std::numeric_limits::quiet_NaN() + * by depth_factor to get a depth in meters, and the values 0 are converted to std::numeric_limits::quiet_NaN() * Otherwise, the image is simply converted to floats * @param in the depth image (if given as short int CV_U, it is assumed to be the depth in millimeters * (as done with the Microsoft Kinect), it is assumed in meters) * @param depth the desired output depth (floats or double) * @param out The rescaled float depth image + * @param depth_factor (optional) factor by which depth is converted to distance (by default = 1000.0 for Kinect sensor) */ CV_EXPORTS_W void - rescaleDepth(InputArray in, int depth, OutputArray out); + rescaleDepth(InputArray in, int depth, OutputArray out, double depth_factor = 1000.0); /** Object that can compute planes in an image */ @@ -564,7 +565,7 @@ namespace rgbd /** Method to compute a transformation from the source frame to the destination one. * Some odometry algorithms do not used some data of frames (eg. ICP does not use images). * In such case corresponding arguments can be set as empty Mat. - * The method returns true if all internal computions were possible (e.g. there were enough correspondences, + * The method returns true if all internal computations were possible (e.g. there were enough correspondences, * system of equations has a solution, etc) and resulting transformation satisfies some test if it's provided * by the Odometry inheritor implementation (e.g. thresholds for maximum translation and rotation). * @param srcImage Image data of the source frame (CV_8UC1) diff --git a/modules/rgbd/include/opencv2/rgbd/dynafu.hpp b/modules/rgbd/include/opencv2/rgbd/dynafu.hpp index d057ebe76c6..55fe36d7e12 100644 --- a/modules/rgbd/include/opencv2/rgbd/dynafu.hpp +++ b/modules/rgbd/include/opencv2/rgbd/dynafu.hpp @@ -10,102 +10,11 @@ #include "opencv2/core.hpp" #include "opencv2/core/affine.hpp" -namespace cv { -namespace dynafu { - -struct CV_EXPORTS_W Params -{ - /** @brief Default parameters - A set of parameters which provides better model quality, can be very slow. - */ - CV_WRAP static Ptr defaultParams(); - - /** @brief Coarse parameters - A set of parameters which provides better speed, can fail to match frames - in case of rapid sensor motion. - */ - CV_WRAP static Ptr coarseParams(); - - /** @brief frame size in pixels */ - CV_PROP_RW Size frameSize; - - /** @brief camera intrinsics */ - CV_PROP Matx33f intr; - - /** @brief pre-scale per 1 meter for input values - - Typical values are: - * 5000 per 1 meter for the 16-bit PNG files of TUM database - * 1000 per 1 meter for Kinect 2 device - * 1 per 1 meter for the 32-bit float images in the ROS bag files - */ - CV_PROP_RW float depthFactor; - - /** @brief Depth sigma in meters for bilateral smooth */ - CV_PROP_RW float bilateral_sigma_depth; - /** @brief Spatial sigma in pixels for bilateral smooth */ - CV_PROP_RW float bilateral_sigma_spatial; - /** @brief Kernel size in pixels for bilateral smooth */ - CV_PROP_RW int bilateral_kernel_size; - - /** @brief Number of pyramid levels for ICP */ - CV_PROP_RW int pyramidLevels; - - /** @brief Resolution of voxel space - - Number of voxels in each dimension. - */ - CV_PROP_RW Vec3i volumeDims; - /** @brief Size of voxel in meters */ - CV_PROP_RW float voxelSize; - - /** @brief Minimal camera movement in meters - - Integrate new depth frame only if camera movement exceeds this value. - */ - CV_PROP_RW float tsdf_min_camera_movement; +#include "kinfu.hpp" - /** @brief initial volume pose in meters */ - Affine3f volumePose; - - /** @brief distance to truncate in meters - - Distances to surface that exceed this value will be truncated to 1.0. - */ - CV_PROP_RW float tsdf_trunc_dist; - - /** @brief max number of frames per voxel - - Each voxel keeps running average of distances no longer than this value. - */ - CV_PROP_RW int tsdf_max_weight; - - /** @brief A length of one raycast step - - How much voxel sizes we skip each raycast step - */ - CV_PROP_RW float raycast_step_factor; - - // gradient delta in voxel sizes - // fixed at 1.0f - // float gradient_delta_factor; - - /** @brief light pose for rendering in meters */ - CV_PROP Vec3f lightPose; - - /** @brief distance theshold for ICP in meters */ - CV_PROP_RW float icpDistThresh; - /** angle threshold for ICP in radians */ - CV_PROP_RW float icpAngleThresh; - /** number of ICP iterations for each pyramid level */ - CV_PROP std::vector icpIterations; - - /** @brief Threshold for depth truncation in meters +namespace cv { - All depth values beyond this threshold will be set to zero - */ - CV_PROP_RW float truncateThreshold; -}; +namespace dynafu { /** @brief DynamicFusion implementation @@ -129,14 +38,19 @@ struct CV_EXPORTS_W Params That's why you need to set the OPENCV_ENABLE_NONFREE option in CMake to use DynamicFusion. */ + + +/** Backwards compatibility for old versions */ +using Params = kinfu::Params; + class CV_EXPORTS_W DynaFu { public: - CV_WRAP static Ptr create(const Ptr& _params); + CV_WRAP static Ptr create(const Ptr& _params); virtual ~DynaFu(); /** @brief Get current parameters */ - virtual const Params& getParams() const = 0; + virtual const kinfu::Params& getParams() const = 0; /** @brief Renders a volume into an image diff --git a/modules/rgbd/include/opencv2/rgbd/kinfu.hpp b/modules/rgbd/include/opencv2/rgbd/kinfu.hpp index 8fcc3189ac3..95f732b6769 100644 --- a/modules/rgbd/include/opencv2/rgbd/kinfu.hpp +++ b/modules/rgbd/include/opencv2/rgbd/kinfu.hpp @@ -24,22 +24,22 @@ struct CV_EXPORTS_W Params /** * @brief Constructor for Params * Sets the initial pose of the TSDF volume. - * @param volumeIntialPoseRot rotation matrix - * @param volumeIntialPoseTransl translation vector + * @param volumeInitialPoseRot rotation matrix + * @param volumeInitialPoseTransl translation vector */ - CV_WRAP Params(Matx33f volumeIntialPoseRot, Vec3f volumeIntialPoseTransl) + CV_WRAP Params(Matx33f volumeInitialPoseRot, Vec3f volumeInitialPoseTransl) { - setInitialVolumePose(volumeIntialPoseRot,volumeIntialPoseTransl); + setInitialVolumePose(volumeInitialPoseRot,volumeInitialPoseTransl); } /** * @brief Constructor for Params * Sets the initial pose of the TSDF volume. - * @param volumeIntialPose 4 by 4 Homogeneous Transform matrix to set the intial pose of TSDF volume + * @param volumeInitialPose 4 by 4 Homogeneous Transform matrix to set the intial pose of TSDF volume */ - CV_WRAP Params(Matx44f volumeIntialPose) + CV_WRAP Params(Matx44f volumeInitialPose) { - setInitialVolumePose(volumeIntialPose); + setInitialVolumePose(volumeInitialPose); } /** @@ -77,7 +77,7 @@ struct CV_EXPORTS_W Params /** @brief frame size in pixels */ CV_PROP_RW Size frameSize; - CV_PROP_RW cv::kinfu::VolumeType volumeType; + CV_PROP_RW kinfu::VolumeType volumeType; /** @brief camera intrinsics */ CV_PROP_RW Matx33f intr; diff --git a/modules/rgbd/include/opencv2/rgbd/large_kinfu.hpp b/modules/rgbd/include/opencv2/rgbd/large_kinfu.hpp new file mode 100644 index 00000000000..7f1428c6d51 --- /dev/null +++ b/modules/rgbd/include/opencv2/rgbd/large_kinfu.hpp @@ -0,0 +1,143 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this +// module's directory + +#ifndef __OPENCV_RGBD_LARGEKINFU_HPP__ +#define __OPENCV_RGBD_LARGEKINFU_HPP__ + +#include + +#include "opencv2/core.hpp" +#include "opencv2/core/affine.hpp" + +namespace cv +{ +namespace large_kinfu +{ +struct CV_EXPORTS_W Params +{ + /** @brief Default parameters + A set of parameters which provides better model quality, can be very slow. + */ + CV_WRAP static Ptr defaultParams(); + + /** @brief Coarse parameters + A set of parameters which provides better speed, can fail to match frames + in case of rapid sensor motion. + */ + CV_WRAP static Ptr coarseParams(); + + /** @brief HashTSDF parameters + A set of parameters suitable for use with HashTSDFVolume + */ + CV_WRAP static Ptr hashTSDFParams(bool isCoarse); + + /** @brief frame size in pixels */ + CV_PROP_RW Size frameSize; + + /** @brief camera intrinsics */ + CV_PROP_RW Matx33f intr; + + /** @brief pre-scale per 1 meter for input values + Typical values are: + * 5000 per 1 meter for the 16-bit PNG files of TUM database + * 1000 per 1 meter for Kinect 2 device + * 1 per 1 meter for the 32-bit float images in the ROS bag files + */ + CV_PROP_RW float depthFactor; + + /** @brief Depth sigma in meters for bilateral smooth */ + CV_PROP_RW float bilateral_sigma_depth; + /** @brief Spatial sigma in pixels for bilateral smooth */ + CV_PROP_RW float bilateral_sigma_spatial; + /** @brief Kernel size in pixels for bilateral smooth */ + CV_PROP_RW int bilateral_kernel_size; + + /** @brief Number of pyramid levels for ICP */ + CV_PROP_RW int pyramidLevels; + + /** @brief Minimal camera movement in meters + Integrate new depth frame only if camera movement exceeds this value. + */ + CV_PROP_RW float tsdf_min_camera_movement; + + /** @brief light pose for rendering in meters */ + CV_PROP_RW Vec3f lightPose; + + /** @brief distance theshold for ICP in meters */ + CV_PROP_RW float icpDistThresh; + /** @brief angle threshold for ICP in radians */ + CV_PROP_RW float icpAngleThresh; + /** @brief number of ICP iterations for each pyramid level */ + CV_PROP_RW std::vector icpIterations; + + /** @brief Threshold for depth truncation in meters + All depth values beyond this threshold will be set to zero + */ + CV_PROP_RW float truncateThreshold; + + /** @brief Volume parameters + */ + kinfu::VolumeParams volumeParams; +}; + +/** @brief Large Scale Dense Depth Fusion implementation + + This class implements a 3d reconstruction algorithm for larger environments using + Spatially hashed TSDF volume "Submaps". + It also runs a periodic posegraph optimization to minimize drift in tracking over long sequences. + Currently the algorithm does not implement a relocalization or loop closure module. + Potentially a Bag of words implementation or RGBD relocalization as described in + Glocker et al. ISMAR 2013 will be implemented + + It takes a sequence of depth images taken from depth sensor + (or any depth images source such as stereo camera matching algorithm or even raymarching + renderer). The output can be obtained as a vector of points and their normals or can be + Phong-rendered from given camera pose. + + An internal representation of a model is a spatially hashed voxel cube that stores TSDF values + which represent the distance to the closest surface (for details read the @cite kinectfusion article + about TSDF). There is no interface to that representation yet. + + For posegraph optimization, a Submap abstraction over the Volume class is created. + New submaps are added to the model when there is low visibility overlap between current viewing frustrum + and the existing volume/model. Multiple submaps are simultaneously tracked and a posegraph is created and + optimized periodically. + + LargeKinfu does not use any OpenCL acceleration yet. + To enable or disable it explicitly use cv::setUseOptimized() or cv::ocl::setUseOpenCL(). + + This implementation is inspired from Kintinuous, InfiniTAM and other SOTA algorithms + + You need to set the OPENCV_ENABLE_NONFREE option in CMake to use KinectFusion. +*/ +class CV_EXPORTS_W LargeKinfu +{ + public: + CV_WRAP static Ptr create(const Ptr& _params); + virtual ~LargeKinfu() = default; + + virtual const Params& getParams() const = 0; + + CV_WRAP virtual void render(OutputArray image, + const Matx44f& cameraPose = Matx44f::eye()) const = 0; + + CV_WRAP virtual void getCloud(OutputArray points, OutputArray normals) const = 0; + + CV_WRAP virtual void getPoints(OutputArray points) const = 0; + + CV_WRAP virtual void getNormals(InputArray points, OutputArray normals) const = 0; + + CV_WRAP virtual void reset() = 0; + + virtual const Affine3f getPose() const = 0; + + CV_WRAP virtual bool update(InputArray depth) = 0; +}; + +} // namespace large_kinfu +} // namespace cv +#endif diff --git a/modules/rgbd/include/opencv2/rgbd/volume.hpp b/modules/rgbd/include/opencv2/rgbd/volume.hpp index 3d10e2dd99c..33f63b1fbfb 100644 --- a/modules/rgbd/include/opencv2/rgbd/volume.hpp +++ b/modules/rgbd/include/opencv2/rgbd/volume.hpp @@ -18,7 +18,7 @@ namespace kinfu class CV_EXPORTS_W Volume { public: - Volume(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor) + Volume(float _voxelSize, Matx44f _pose, float _raycastStepFactor) : voxelSize(_voxelSize), voxelSizeInv(1.0f / voxelSize), pose(_pose), @@ -28,19 +28,18 @@ class CV_EXPORTS_W Volume virtual ~Volume(){}; - virtual void integrate(InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, - const cv::kinfu::Intr& intrinsics) = 0; - virtual void raycast(const cv::Matx44f& cameraPose, const cv::kinfu::Intr& intrinsics, - cv::Size frameSize, cv::OutputArray points, - cv::OutputArray normals) const = 0; - virtual void fetchNormals(cv::InputArray points, cv::OutputArray _normals) const = 0; - virtual void fetchPointsNormals(cv::OutputArray points, cv::OutputArray normals) const = 0; - virtual void reset() = 0; + virtual void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, + const kinfu::Intr& intrinsics, const int frameId = 0) = 0; + virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, + const Size& frameSize, OutputArray points, OutputArray normals) const = 0; + virtual void fetchNormals(InputArray points, OutputArray _normals) const = 0; + virtual void fetchPointsNormals(OutputArray points, OutputArray normals) const = 0; + virtual void reset() = 0; public: const float voxelSize; const float voxelSizeInv; - const cv::Affine3f pose; + const Affine3f pose; const float raycastStepFactor; }; @@ -50,9 +49,70 @@ enum class VolumeType HASHTSDF = 1 }; -CV_EXPORTS_W cv::Ptr makeVolume(VolumeType _volumeType, float _voxelSize, cv::Matx44f _pose, - float _raycastStepFactor, float _truncDist, int _maxWeight, - float _truncateThreshold, Vec3i _resolution); +struct CV_EXPORTS_W VolumeParams +{ + /** @brief Type of Volume + Values can be TSDF (single volume) or HASHTSDF (hashtable of volume units) + */ + CV_PROP_RW VolumeType type; + + /** @brief Resolution of voxel space + Number of voxels in each dimension. + Applicable only for TSDF Volume. + HashTSDF volume only supports equal resolution in all three dimensions + */ + CV_PROP_RW Vec3i resolution; + + /** @brief Resolution of volumeUnit in voxel space + Number of voxels in each dimension for volumeUnit + Applicable only for hashTSDF. + */ + CV_PROP_RW int unitResolution = {0}; + + /** @brief Initial pose of the volume in meters */ + Affine3f pose; + + /** @brief Length of voxels in meters */ + CV_PROP_RW float voxelSize; + + /** @brief TSDF truncation distance + Distances greater than value from surface will be truncated to 1.0 + */ + CV_PROP_RW float tsdfTruncDist; + + /** @brief Max number of frames to integrate per voxel + Represents the max number of frames over which a running average + of the TSDF is calculated for a voxel + */ + CV_PROP_RW int maxWeight; + + /** @brief Threshold for depth truncation in meters + Truncates the depth greater than threshold to 0 + */ + CV_PROP_RW float depthTruncThreshold; + + /** @brief Length of single raycast step + Describes the percentage of voxel length that is skipped per march + */ + CV_PROP_RW float raycastStepFactor; + + /** @brief Default set of parameters that provide higher quality reconstruction + at the cost of slow performance. + */ + CV_WRAP static Ptr defaultParams(VolumeType _volumeType); + + /** @brief Coarse set of parameters that provides relatively higher performance + at the cost of reconstrution quality. + */ + CV_WRAP static Ptr coarseParams(VolumeType _volumeType); +}; + + +Ptr makeVolume(const VolumeParams& _volumeParams); +CV_EXPORTS_W Ptr makeVolume(VolumeType _volumeType, float _voxelSize, Matx44f _pose, + float _raycastStepFactor, float _truncDist, int _maxWeight, + float _truncateThreshold, Vec3i _resolution); + } // namespace kinfu } // namespace cv #endif diff --git a/modules/rgbd/samples/dynafu_demo.cpp b/modules/rgbd/samples/dynafu_demo.cpp index b69d8725f9c..8b27021ea5a 100644 --- a/modules/rgbd/samples/dynafu_demo.cpp +++ b/modules/rgbd/samples/dynafu_demo.cpp @@ -13,208 +13,16 @@ #include #include #include +#include "io_utils.hpp" using namespace cv; using namespace cv::dynafu; -using namespace std; +using namespace cv::io_utils; #ifdef HAVE_OPENCV_VIZ #include #endif -static vector readDepth(std::string fileList); - -static vector readDepth(std::string fileList) -{ - vector v; - - fstream file(fileList); - if(!file.is_open()) - throw std::runtime_error("Failed to read depth list"); - - std::string dir; - size_t slashIdx = fileList.rfind('/'); - slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); - dir = fileList.substr(0, slashIdx); - - while(!file.eof()) - { - std::string s, imgPath; - std::getline(file, s); - if(s.empty() || s[0] == '#') continue; - std::stringstream ss; - ss << s; - double thumb; - ss >> thumb >> imgPath; - v.push_back(dir+'/'+imgPath); - } - - return v; -} - -struct DepthWriter -{ - DepthWriter(string fileList) : - file(fileList, ios::out), count(0), dir() - { - size_t slashIdx = fileList.rfind('/'); - slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); - dir = fileList.substr(0, slashIdx); - - if(!file.is_open()) - throw std::runtime_error("Failed to write depth list"); - - file << "# depth maps saved from device" << endl; - file << "# useless_number filename" << endl; - } - - void append(InputArray _depth) - { - Mat depth = _depth.getMat(); - string depthFname = cv::format("%04d.png", count); - string fullDepthFname = dir + '/' + depthFname; - if(!imwrite(fullDepthFname, depth)) - throw std::runtime_error("Failed to write depth to file " + fullDepthFname); - file << count++ << " " << depthFname << endl; - } - - fstream file; - int count; - string dir; -}; - -namespace Kinect2Params -{ - static const Size frameSize = Size(512, 424); - // approximate values, no guarantee to be correct - static const float focal = 366.1f; - static const float cx = 258.2f; - static const float cy = 204.f; - static const float k1 = 0.12f; - static const float k2 = -0.34f; - static const float k3 = 0.12f; -}; - -struct DepthSource -{ -public: - DepthSource(int cam) : - DepthSource("", cam) - { } - - DepthSource(String fileListName) : - DepthSource(fileListName, -1) - { } - - DepthSource(String fileListName, int cam) : - depthFileList(fileListName.empty() ? vector() : readDepth(fileListName)), - frameIdx(0), - vc( cam >= 0 ? VideoCapture(VideoCaptureAPIs::CAP_OPENNI2 + cam) : VideoCapture()), - undistortMap1(), - undistortMap2(), - useKinect2Workarounds(true) - { - } - - UMat getDepth() - { - UMat out; - if (!vc.isOpened()) - { - if (frameIdx < depthFileList.size()) - { - Mat f = cv::imread(depthFileList[frameIdx++], IMREAD_ANYDEPTH); - f.copyTo(out); - } - else - { - return UMat(); - } - } - else - { - vc.grab(); - vc.retrieve(out, CAP_OPENNI_DEPTH_MAP); - - // workaround for Kinect 2 - if(useKinect2Workarounds) - { - out = out(Rect(Point(), Kinect2Params::frameSize)); - - UMat outCopy; - // linear remap adds gradient between valid and invalid pixels - // which causes garbage, use nearest instead - remap(out, outCopy, undistortMap1, undistortMap2, cv::INTER_NEAREST); - - cv::flip(outCopy, out, 1); - } - } - if (out.empty()) - throw std::runtime_error("Matrix is empty"); - return out; - } - - bool empty() - { - return depthFileList.empty() && !(vc.isOpened()); - } - - void updateParams(Params& params) - { - if (vc.isOpened()) - { - // this should be set in according to user's depth sensor - int w = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_WIDTH); - int h = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_HEIGHT); - - float focal = (float)vc.get(CAP_OPENNI_DEPTH_GENERATOR | CAP_PROP_OPENNI_FOCAL_LENGTH); - - // it's recommended to calibrate sensor to obtain its intrinsics - float fx, fy, cx, cy; - Size frameSize; - if(useKinect2Workarounds) - { - fx = fy = Kinect2Params::focal; - cx = Kinect2Params::cx; - cy = Kinect2Params::cy; - - frameSize = Kinect2Params::frameSize; - } - else - { - fx = fy = focal; - cx = w/2 - 0.5f; - cy = h/2 - 0.5f; - - frameSize = Size(w, h); - } - - Matx33f camMatrix = Matx33f(fx, 0, cx, - 0, fy, cy, - 0, 0, 1); - - params.frameSize = frameSize; - params.intr = camMatrix; - params.depthFactor = 1000.f; - - Matx distCoeffs; - distCoeffs(0) = Kinect2Params::k1; - distCoeffs(1) = Kinect2Params::k2; - distCoeffs(4) = Kinect2Params::k3; - if(useKinect2Workarounds) - initUndistortRectifyMap(camMatrix, distCoeffs, cv::noArray(), - camMatrix, frameSize, CV_16SC2, - undistortMap1, undistortMap2); - } - } - - vector depthFileList; - size_t frameIdx; - VideoCapture vc; - UMat undistortMap1, undistortMap2; - bool useKinect2Workarounds; -}; - #ifdef HAVE_OPENCV_VIZ const std::string vizWindowName = "cloud"; @@ -265,7 +73,7 @@ int main(int argc, char **argv) { bool coarse = false; bool idle = false; - string recordPath; + std::string recordPath; CommandLineParser parser(argc, argv, keys); parser.about(message); @@ -312,13 +120,13 @@ int main(int argc, char **argv) if(!recordPath.empty()) depthWriter = makePtr(recordPath); - Ptr params; + Ptr params; Ptr df; if(coarse) - params = Params::coarseParams(); + params = kinfu::Params::coarseParams(); else - params = Params::defaultParams(); + params = kinfu::Params::defaultParams(); // These params can be different for each depth sensor ds->updateParams(*params); diff --git a/modules/rgbd/samples/io_utils.hpp b/modules/rgbd/samples/io_utils.hpp new file mode 100644 index 00000000000..c96d6c5345d --- /dev/null +++ b/modules/rgbd/samples/io_utils.hpp @@ -0,0 +1,313 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_RGBS_IO_UTILS_HPP +#define OPENCV_RGBS_IO_UTILS_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace cv +{ +namespace io_utils +{ + +static std::vector readDepth(const std::string& fileList) +{ + std::vector v; + + std::fstream file(fileList); + if (!file.is_open()) + throw std::runtime_error("Failed to read depth list"); + + std::string dir; + size_t slashIdx = fileList.rfind('/'); + slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); + dir = fileList.substr(0, slashIdx); + + while (!file.eof()) + { + std::string s, imgPath; + std::getline(file, s); + if (s.empty() || s[0] == '#') + continue; + std::stringstream ss; + ss << s; + double thumb; + ss >> thumb >> imgPath; + v.push_back(dir + '/' + imgPath); + } + + return v; +} + +struct DepthWriter +{ + DepthWriter(std::string fileList) : file(fileList, std::ios::out), count(0), dir() + { + size_t slashIdx = fileList.rfind('/'); + slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); + dir = fileList.substr(0, slashIdx); + + if (!file.is_open()) + throw std::runtime_error("Failed to write depth list"); + + file << "# depth maps saved from device" << std::endl; + file << "# useless_number filename" << std::endl; + } + + void append(InputArray _depth) + { + Mat depth = _depth.getMat(); + std::string depthFname = cv::format("%04d.png", count); + std::string fullDepthFname = dir + '/' + depthFname; + if (!imwrite(fullDepthFname, depth)) + throw std::runtime_error("Failed to write depth to file " + fullDepthFname); + file << count++ << " " << depthFname << std::endl; + } + + std::fstream file; + int count; + std::string dir; +}; + +namespace Kinect2Params +{ +static const Size frameSize = Size(512, 424); +// approximate values, no guarantee to be correct +static const float focal = 366.1f; +static const float cx = 258.2f; +static const float cy = 204.f; +static const float k1 = 0.12f; +static const float k2 = -0.34f; +static const float k3 = 0.12f; +}; // namespace Kinect2Params + +struct DepthSource +{ + public: + enum Type + { + DEPTH_LIST, + DEPTH_KINECT2_LIST, + DEPTH_KINECT2, + DEPTH_REALSENSE + }; + + DepthSource(int cam) : DepthSource("", cam) {} + + DepthSource(String fileListName) : DepthSource(fileListName, -1) {} + + DepthSource(String fileListName, int cam) + : depthFileList(fileListName.empty() ? std::vector() + : readDepth(fileListName)), + frameIdx(0), + undistortMap1(), + undistortMap2() + { + if (cam >= 0) + { + vc = VideoCapture(VideoCaptureAPIs::CAP_OPENNI2 + cam); + if (vc.isOpened()) + { + sourceType = Type::DEPTH_KINECT2; + } + else + { + vc = VideoCapture(VideoCaptureAPIs::CAP_REALSENSE + cam); + if (vc.isOpened()) + { + sourceType = Type::DEPTH_REALSENSE; + } + } + } + else + { + vc = VideoCapture(); + sourceType = Type::DEPTH_KINECT2_LIST; + } + } + + UMat getDepth() + { + UMat out; + if (!vc.isOpened()) + { + if (frameIdx < depthFileList.size()) + { + Mat f = cv::imread(depthFileList[frameIdx++], IMREAD_ANYDEPTH); + f.copyTo(out); + } + else + { + return UMat(); + } + } + else + { + vc.grab(); + switch (sourceType) + { + case Type::DEPTH_KINECT2: vc.retrieve(out, CAP_OPENNI_DEPTH_MAP); break; + case Type::DEPTH_REALSENSE: vc.retrieve(out, CAP_INTELPERC_DEPTH_MAP); break; + default: + // unknown depth source + vc.retrieve(out); + } + + // workaround for Kinect 2 + if (sourceType == Type::DEPTH_KINECT2) + { + out = out(Rect(Point(), Kinect2Params::frameSize)); + + UMat outCopy; + // linear remap adds gradient between valid and invalid pixels + // which causes garbage, use nearest instead + remap(out, outCopy, undistortMap1, undistortMap2, cv::INTER_NEAREST); + + cv::flip(outCopy, out, 1); + } + } + if (out.empty()) + throw std::runtime_error("Matrix is empty"); + return out; + } + + bool empty() { return depthFileList.empty() && !(vc.isOpened()); } + + void updateIntrinsics(Matx33f& _intrinsics, Size& _frameSize, float& _depthFactor) + { + if (vc.isOpened()) + { + // this should be set in according to user's depth sensor + int w = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_WIDTH); + int h = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_HEIGHT); + + // it's recommended to calibrate sensor to obtain its intrinsics + float fx, fy, cx, cy; + float depthFactor = 1000.f; + Size frameSize; + if (sourceType == Type::DEPTH_KINECT2) + { + fx = fy = Kinect2Params::focal; + cx = Kinect2Params::cx; + cy = Kinect2Params::cy; + + frameSize = Kinect2Params::frameSize; + } + else + { + if (sourceType == Type::DEPTH_REALSENSE) + { + fx = (float)vc.get(CAP_PROP_INTELPERC_DEPTH_FOCAL_LENGTH_HORZ); + fy = (float)vc.get(CAP_PROP_INTELPERC_DEPTH_FOCAL_LENGTH_VERT); + depthFactor = 1.f / (float)vc.get(CAP_PROP_INTELPERC_DEPTH_SATURATION_VALUE); + } + else + { + fx = fy = + (float)vc.get(CAP_OPENNI_DEPTH_GENERATOR | CAP_PROP_OPENNI_FOCAL_LENGTH); + } + + cx = w / 2 - 0.5f; + cy = h / 2 - 0.5f; + + frameSize = Size(w, h); + } + + Matx33f camMatrix = Matx33f(fx, 0, cx, 0, fy, cy, 0, 0, 1); + _intrinsics = camMatrix; + _frameSize = frameSize; + _depthFactor = depthFactor; + } + } + + void updateVolumeParams(const Vec3i& _resolution, float& _voxelSize, float& _tsdfTruncDist, + Affine3f& _volumePose, float& _depthTruncateThreshold) + { + float volumeSize = 3.0f; + _depthTruncateThreshold = 0.0f; + // RealSense has shorter depth range, some params should be tuned + if (sourceType == Type::DEPTH_REALSENSE) + { + volumeSize = 1.f; + _voxelSize = volumeSize / _resolution[0]; + _tsdfTruncDist = 0.01f; + _depthTruncateThreshold = 2.5f; + } + _volumePose = Affine3f().translate(Vec3f(-volumeSize / 2.f, -volumeSize / 2.f, 0.05f)); + } + + void updateICPParams(float& _icpDistThresh, float& _bilateralSigmaDepth) + { + _icpDistThresh = 0.1f; + _bilateralSigmaDepth = 0.04f; + // RealSense has shorter depth range, some params should be tuned + if (sourceType == Type::DEPTH_REALSENSE) + { + _icpDistThresh = 0.01f; + _bilateralSigmaDepth = 0.01f; + } + } + + void updateParams(large_kinfu::Params& params) + { + if (vc.isOpened()) + { + updateIntrinsics(params.intr, params.frameSize, params.depthFactor); + updateVolumeParams(params.volumeParams.resolution, params.volumeParams.voxelSize, + params.volumeParams.tsdfTruncDist, params.volumeParams.pose, + params.truncateThreshold); + updateICPParams(params.icpDistThresh, params.bilateral_sigma_depth); + + if (sourceType == Type::DEPTH_KINECT2) + { + Matx distCoeffs; + distCoeffs(0) = Kinect2Params::k1; + distCoeffs(1) = Kinect2Params::k2; + distCoeffs(4) = Kinect2Params::k3; + + initUndistortRectifyMap(params.intr, distCoeffs, cv::noArray(), params.intr, + params.frameSize, CV_16SC2, undistortMap1, undistortMap2); + } + } + } + + void updateParams(kinfu::Params& params) + { + if (vc.isOpened()) + { + updateIntrinsics(params.intr, params.frameSize, params.depthFactor); + updateVolumeParams(params.volumeDims, params.voxelSize, + params.tsdf_trunc_dist, params.volumePose, params.truncateThreshold); + updateICPParams(params.icpDistThresh, params.bilateral_sigma_depth); + + if (sourceType == Type::DEPTH_KINECT2) + { + Matx distCoeffs; + distCoeffs(0) = Kinect2Params::k1; + distCoeffs(1) = Kinect2Params::k2; + distCoeffs(4) = Kinect2Params::k3; + + initUndistortRectifyMap(params.intr, distCoeffs, cv::noArray(), params.intr, + params.frameSize, CV_16SC2, undistortMap1, undistortMap2); + } + } + } + + std::vector depthFileList; + size_t frameIdx; + VideoCapture vc; + UMat undistortMap1, undistortMap2; + Type sourceType; +}; +} // namespace io_utils + +} // namespace cv +#endif /* ifndef OPENCV_RGBS_IO_UTILS_HPP */ diff --git a/modules/rgbd/samples/kinfu_demo.cpp b/modules/rgbd/samples/kinfu_demo.cpp index 147ffaa2d8e..e264ba38bbe 100644 --- a/modules/rgbd/samples/kinfu_demo.cpp +++ b/modules/rgbd/samples/kinfu_demo.cpp @@ -11,272 +11,16 @@ #include #include +#include "io_utils.hpp" + using namespace cv; using namespace cv::kinfu; -using namespace std; +using namespace cv::io_utils; #ifdef HAVE_OPENCV_VIZ #include #endif -static vector readDepth(std::string fileList); - -static vector readDepth(std::string fileList) -{ - vector v; - - fstream file(fileList); - if(!file.is_open()) - throw std::runtime_error("Failed to read depth list"); - - std::string dir; - size_t slashIdx = fileList.rfind('/'); - slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); - dir = fileList.substr(0, slashIdx); - - while(!file.eof()) - { - std::string s, imgPath; - std::getline(file, s); - if(s.empty() || s[0] == '#') continue; - std::stringstream ss; - ss << s; - double thumb; - ss >> thumb >> imgPath; - v.push_back(dir+'/'+imgPath); - } - - return v; -} - -struct DepthWriter -{ - DepthWriter(string fileList) : - file(fileList, ios::out), count(0), dir() - { - size_t slashIdx = fileList.rfind('/'); - slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); - dir = fileList.substr(0, slashIdx); - - if(!file.is_open()) - throw std::runtime_error("Failed to write depth list"); - - file << "# depth maps saved from device" << endl; - file << "# useless_number filename" << endl; - } - - void append(InputArray _depth) - { - Mat depth = _depth.getMat(); - string depthFname = cv::format("%04d.png", count); - string fullDepthFname = dir + '/' + depthFname; - if(!imwrite(fullDepthFname, depth)) - throw std::runtime_error("Failed to write depth to file " + fullDepthFname); - file << count++ << " " << depthFname << endl; - } - - fstream file; - int count; - string dir; -}; - -namespace Kinect2Params -{ - static const Size frameSize = Size(512, 424); - // approximate values, no guarantee to be correct - static const float focal = 366.1f; - static const float cx = 258.2f; - static const float cy = 204.f; - static const float k1 = 0.12f; - static const float k2 = -0.34f; - static const float k3 = 0.12f; -}; - -struct DepthSource -{ -public: - enum Type - { - DEPTH_LIST, - DEPTH_KINECT2_LIST, - DEPTH_KINECT2, - DEPTH_REALSENSE - }; - - DepthSource(int cam) : - DepthSource("", cam) - { } - - DepthSource(String fileListName) : - DepthSource(fileListName, -1) - { } - - DepthSource(String fileListName, int cam) : - depthFileList(fileListName.empty() ? vector() : readDepth(fileListName)), - frameIdx(0), - undistortMap1(), - undistortMap2() - { - if(cam >= 0) - { - vc = VideoCapture(VideoCaptureAPIs::CAP_OPENNI2 + cam); - if(vc.isOpened()) - { - sourceType = Type::DEPTH_KINECT2; - } - else - { - vc = VideoCapture(VideoCaptureAPIs::CAP_REALSENSE + cam); - if(vc.isOpened()) - { - sourceType = Type::DEPTH_REALSENSE; - } - } - } - else - { - vc = VideoCapture(); - sourceType = Type::DEPTH_KINECT2_LIST; - } - } - - UMat getDepth() - { - UMat out; - if (!vc.isOpened()) - { - if (frameIdx < depthFileList.size()) - { - Mat f = cv::imread(depthFileList[frameIdx++], IMREAD_ANYDEPTH); - f.copyTo(out); - } - else - { - return UMat(); - } - } - else - { - vc.grab(); - switch (sourceType) - { - case Type::DEPTH_KINECT2: - vc.retrieve(out, CAP_OPENNI_DEPTH_MAP); - break; - case Type::DEPTH_REALSENSE: - vc.retrieve(out, CAP_INTELPERC_DEPTH_MAP); - break; - default: - // unknown depth source - vc.retrieve(out); - } - - // workaround for Kinect 2 - if(sourceType == Type::DEPTH_KINECT2) - { - out = out(Rect(Point(), Kinect2Params::frameSize)); - - UMat outCopy; - // linear remap adds gradient between valid and invalid pixels - // which causes garbage, use nearest instead - remap(out, outCopy, undistortMap1, undistortMap2, cv::INTER_NEAREST); - - cv::flip(outCopy, out, 1); - } - } - if (out.empty()) - throw std::runtime_error("Matrix is empty"); - return out; - } - - bool empty() - { - return depthFileList.empty() && !(vc.isOpened()); - } - - void updateParams(Params& params) - { - if (vc.isOpened()) - { - // this should be set in according to user's depth sensor - int w = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_WIDTH); - int h = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_HEIGHT); - - // it's recommended to calibrate sensor to obtain its intrinsics - float fx, fy, cx, cy; - float depthFactor = 1000.f; - Size frameSize; - if(sourceType == Type::DEPTH_KINECT2) - { - fx = fy = Kinect2Params::focal; - cx = Kinect2Params::cx; - cy = Kinect2Params::cy; - - frameSize = Kinect2Params::frameSize; - } - else - { - if(sourceType == Type::DEPTH_REALSENSE) - { - fx = (float)vc.get(CAP_PROP_INTELPERC_DEPTH_FOCAL_LENGTH_HORZ); - fy = (float)vc.get(CAP_PROP_INTELPERC_DEPTH_FOCAL_LENGTH_VERT); - depthFactor = 1.f/(float)vc.get(CAP_PROP_INTELPERC_DEPTH_SATURATION_VALUE); - } - else - { - fx = fy = (float)vc.get(CAP_OPENNI_DEPTH_GENERATOR | CAP_PROP_OPENNI_FOCAL_LENGTH); - } - - cx = w/2 - 0.5f; - cy = h/2 - 0.5f; - - frameSize = Size(w, h); - } - - Matx33f camMatrix = Matx33f(fx, 0, cx, - 0, fy, cy, - 0, 0, 1); - - params.frameSize = frameSize; - params.intr = camMatrix; - params.depthFactor = depthFactor; - - // RealSense has shorter depth range, some params should be tuned - if(sourceType == Type::DEPTH_REALSENSE) - { - // all sizes in meters - float cubeSize = 1.f; - params.voxelSize = cubeSize/params.volumeDims[0]; - params.tsdf_trunc_dist = 0.01f; - params.icpDistThresh = 0.01f; - params.volumePose = Affine3f().translate(Vec3f(-cubeSize/2.f, - -cubeSize/2.f, - 0.05f)); - params.truncateThreshold = 2.5f; - params.bilateral_sigma_depth = 0.01f; - } - - if(sourceType == Type::DEPTH_KINECT2) - { - Matx distCoeffs; - distCoeffs(0) = Kinect2Params::k1; - distCoeffs(1) = Kinect2Params::k2; - distCoeffs(4) = Kinect2Params::k3; - - initUndistortRectifyMap(camMatrix, distCoeffs, cv::noArray(), - camMatrix, frameSize, CV_16SC2, - undistortMap1, undistortMap2); - } - } - } - - vector depthFileList; - size_t frameIdx; - VideoCapture vc; - UMat undistortMap1, undistortMap2; - Type sourceType; -}; - #ifdef HAVE_OPENCV_VIZ const std::string vizWindowName = "cloud"; @@ -329,7 +73,7 @@ int main(int argc, char **argv) bool coarse = false; bool idle = false; bool useHashTSDF = false; - string recordPath; + std::string recordPath; CommandLineParser parser(argc, argv, keys); parser.about(message); diff --git a/modules/rgbd/samples/large_kinfu_demo.cpp b/modules/rgbd/samples/large_kinfu_demo.cpp new file mode 100644 index 00000000000..40d14ca8cae --- /dev/null +++ b/modules/rgbd/samples/large_kinfu_demo.cpp @@ -0,0 +1,260 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this +// module's directory + +#include +#include +#include +#include +#include +#include + +#include "io_utils.hpp" + +using namespace cv; +using namespace cv::kinfu; +using namespace cv::large_kinfu; +using namespace cv::io_utils; + +#ifdef HAVE_OPENCV_VIZ +#include +#endif + +#ifdef HAVE_OPENCV_VIZ +const std::string vizWindowName = "cloud"; + +struct PauseCallbackArgs +{ + PauseCallbackArgs(LargeKinfu& _largeKinfu) : largeKinfu(_largeKinfu) {} + + LargeKinfu& largeKinfu; +}; + +void pauseCallback(const viz::MouseEvent& me, void* args); +void pauseCallback(const viz::MouseEvent& me, void* args) +{ + if (me.type == viz::MouseEvent::Type::MouseMove || + me.type == viz::MouseEvent::Type::MouseScrollDown || + me.type == viz::MouseEvent::Type::MouseScrollUp) + { + PauseCallbackArgs pca = *((PauseCallbackArgs*)(args)); + viz::Viz3d window(vizWindowName); + UMat rendered; + pca.largeKinfu.render(rendered, window.getViewerPose().matrix); + imshow("render", rendered); + waitKey(1); + } +} +#endif + +static const char* keys = { + "{help h usage ? | | print this message }" + "{depth | | Path to depth.txt file listing a set of depth images }" + "{camera |0| Index of depth camera to be used as a depth source }" + "{coarse | | Run on coarse settings (fast but ugly) or on default (slow but looks better)," + " in coarse mode points and normals are displayed }" + "{idle | | Do not run LargeKinfu, just display depth frames }" + "{record | | Write depth frames to specified file list" + " (the same format as for the 'depth' key) }" +}; + +static const std::string message = + "\nThis demo uses live depth input or RGB-D dataset taken from" + "\nhttps://vision.in.tum.de/data/datasets/rgbd-dataset" + "\nto demonstrate Submap based large environment reconstruction" + "\nThis module uses the newer hashtable based TSDFVolume (relatively fast) for larger " + "reconstructions by default\n"; + +int main(int argc, char** argv) +{ + bool coarse = false; + bool idle = false; + std::string recordPath; + + CommandLineParser parser(argc, argv, keys); + parser.about(message); + + if (!parser.check()) + { + parser.printMessage(); + parser.printErrors(); + return -1; + } + + if (parser.has("help")) + { + parser.printMessage(); + return 0; + } + if (parser.has("coarse")) + { + coarse = true; + } + if (parser.has("record")) + { + recordPath = parser.get("record"); + } + if (parser.has("idle")) + { + idle = true; + } + + Ptr ds; + if (parser.has("depth")) + ds = makePtr(parser.get("depth")); + else + ds = makePtr(parser.get("camera")); + + if (ds->empty()) + { + std::cerr << "Failed to open depth source" << std::endl; + parser.printMessage(); + return -1; + } + + Ptr depthWriter; + if (!recordPath.empty()) + depthWriter = makePtr(recordPath); + + Ptr params; + Ptr largeKinfu; + + params = large_kinfu::Params::hashTSDFParams(coarse); + + // These params can be different for each depth sensor + ds->updateParams(*params); + + // Disabled until there is no OpenCL accelerated HashTSDF is available + cv::setUseOptimized(false); + + if (!idle) + largeKinfu = LargeKinfu::create(params); + +#ifdef HAVE_OPENCV_VIZ + cv::viz::Viz3d window(vizWindowName); + window.setViewerPose(Affine3f::Identity()); + bool pause = false; +#endif + + UMat rendered; + UMat points; + UMat normals; + + int64 prevTime = getTickCount(); + + for (UMat frame = ds->getDepth(); !frame.empty(); frame = ds->getDepth()) + { + if (depthWriter) + depthWriter->append(frame); + +#ifdef HAVE_OPENCV_VIZ + if (pause) + { + // doesn't happen in idle mode + largeKinfu->getCloud(points, normals); + if (!points.empty() && !normals.empty()) + { + viz::WCloud cloudWidget(points, viz::Color::white()); + viz::WCloudNormals cloudNormals(points, normals, /*level*/ 1, /*scale*/ 0.05, + viz::Color::gray()); + window.showWidget("cloud", cloudWidget); + window.showWidget("normals", cloudNormals); + + Vec3d volSize = largeKinfu->getParams().volumeParams.voxelSize * + Vec3d(largeKinfu->getParams().volumeParams.resolution); + window.showWidget("cube", viz::WCube(Vec3d::all(0), volSize), + largeKinfu->getParams().volumeParams.pose); + PauseCallbackArgs pca(*largeKinfu); + window.registerMouseCallback(pauseCallback, (void*)&pca); + window.showWidget("text", + viz::WText(cv::String("Move camera in this window. " + "Close the window or press Q to resume"), + Point())); + window.spin(); + window.removeWidget("text"); + window.removeWidget("cloud"); + window.removeWidget("normals"); + window.registerMouseCallback(0); + } + + pause = false; + } + else +#endif + { + UMat cvt8; + float depthFactor = params->depthFactor; + convertScaleAbs(frame, cvt8, 0.25 * 256. / depthFactor); + if (!idle) + { + imshow("depth", cvt8); + + if (!largeKinfu->update(frame)) + { + largeKinfu->reset(); + std::cout << "reset" << std::endl; + } +#ifdef HAVE_OPENCV_VIZ + else + { + if (coarse) + { + largeKinfu->getCloud(points, normals); + if (!points.empty() && !normals.empty()) + { + viz::WCloud cloudWidget(points, viz::Color::white()); + viz::WCloudNormals cloudNormals(points, normals, /*level*/ 1, + /*scale*/ 0.05, viz::Color::gray()); + window.showWidget("cloud", cloudWidget); + window.showWidget("normals", cloudNormals); + } + } + + // window.showWidget("worldAxes", viz::WCoordinateSystem()); + Vec3d volSize = largeKinfu->getParams().volumeParams.voxelSize * + largeKinfu->getParams().volumeParams.resolution; + window.showWidget("cube", viz::WCube(Vec3d::all(0), volSize), + largeKinfu->getParams().volumeParams.pose); + window.setViewerPose(largeKinfu->getPose()); + window.spinOnce(1, true); + } +#endif + + largeKinfu->render(rendered); + } + else + { + rendered = cvt8; + } + } + + int64 newTime = getTickCount(); + putText(rendered, + cv::format("FPS: %2d press R to reset, P to pause, Q to quit", + (int)(getTickFrequency() / (newTime - prevTime))), + Point(0, rendered.rows - 1), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 255)); + prevTime = newTime; + imshow("render", rendered); + + int c = waitKey(1); + switch (c) + { + case 'r': + if (!idle) + largeKinfu->reset(); + break; + case 'q': return 0; +#ifdef HAVE_OPENCV_VIZ + case 'p': + if (!idle) + pause = true; +#endif + default: break; + } + } + + return 0; +} diff --git a/modules/rgbd/src/dynafu.cpp b/modules/rgbd/src/dynafu.cpp index d663275d32d..002be7bd6d7 100644 --- a/modules/rgbd/src/dynafu.cpp +++ b/modules/rgbd/src/dynafu.cpp @@ -82,76 +82,6 @@ namespace cv { namespace dynafu { using namespace kinfu; -Ptr Params::defaultParams() -{ - Params p; - - p.frameSize = Size(640, 480); - - float fx, fy, cx, cy; - fx = fy = 525.f; - cx = p.frameSize.width/2 - 0.5f; - cy = p.frameSize.height/2 - 0.5f; - p.intr = Matx33f(fx, 0, cx, - 0, fy, cy, - 0, 0, 1); - - // 5000 for the 16-bit PNG files - // 1 for the 32-bit float images in the ROS bag files - p.depthFactor = 5000; - - // sigma_depth is scaled by depthFactor when calling bilateral filter - p.bilateral_sigma_depth = 0.04f; //meter - p.bilateral_sigma_spatial = 4.5; //pixels - p.bilateral_kernel_size = 7; //pixels - - p.icpAngleThresh = (float)(30. * CV_PI / 180.); // radians - p.icpDistThresh = 0.1f; // meters - - p.icpIterations = {10, 5, 4}; - p.pyramidLevels = (int)p.icpIterations.size(); - - p.tsdf_min_camera_movement = 0.f; //meters, disabled - - p.volumeDims = Vec3i::all(512); //number of voxels - - float volSize = 3.f; - p.voxelSize = volSize/512.f; //meters - - // default pose of volume cube - p.volumePose = Affine3f().translate(Vec3f(-volSize/2.f, -volSize/2.f, 0.5f)); - p.tsdf_trunc_dist = 0.04f; //meters; - p.tsdf_max_weight = 64; //frames - - p.raycast_step_factor = 0.25f; //in voxel sizes - // gradient delta factor is fixed at 1.0f and is not used - //p.gradient_delta_factor = 0.5f; //in voxel sizes - - //p.lightPose = p.volume_pose.translation()/4; //meters - p.lightPose = Vec3f::all(0.f); //meters - - // depth truncation is not used by default but can be useful in some scenes - p.truncateThreshold = 0.f; //meters - - return makePtr(p); -} - -Ptr Params::coarseParams() -{ - Ptr p = defaultParams(); - - p->icpIterations = {5, 3, 2}; - p->pyramidLevels = (int)p->icpIterations.size(); - - float volSize = 3.f; - p->volumeDims = Vec3i::all(128); //number of voxels - p->voxelSize = volSize/128.f; - - p->raycast_step_factor = 0.75f; //in voxel sizes - - return p; -} - // T should be Mat or UMat template< typename T > class DynaFuImpl : public DynaFu diff --git a/modules/rgbd/src/fast_icp.cpp b/modules/rgbd/src/fast_icp.cpp index 06d4a750647..ab25eb94f0d 100644 --- a/modules/rgbd/src/fast_icp.cpp +++ b/modules/rgbd/src/fast_icp.cpp @@ -32,7 +32,6 @@ class ICPImpl : public ICP InputArray oldPoints, InputArray oldNormals, InputArray newPoints, InputArray newNormals ) const override; - template < typename T > bool estimateTransformT(cv::Affine3f& transform, const vector& oldPoints, const vector& oldNormals, @@ -298,7 +297,6 @@ struct GetAbInvoker : ParallelLoopBody continue; // build point-wise vector ab = [ A | b ] - v_float32x4 VxNv = crossProduct(newP, oldN); Point3f VxN; VxN.x = VxNv.get0(); @@ -449,7 +447,6 @@ struct GetAbInvoker : ParallelLoopBody //try to optimize Point3f VxN = newP.cross(oldN); float ab[7] = {VxN.x, VxN.y, VxN.z, oldN.x, oldN.y, oldN.z, oldN.dot(-diff)}; - // build point-wise upper-triangle matrix [ab^T * ab] w/o last row // which is [A^T*A | A^T*b] // and gather sum diff --git a/modules/rgbd/src/fast_icp.hpp b/modules/rgbd/src/fast_icp.hpp index 7a9f0694096..a11a919da3e 100644 --- a/modules/rgbd/src/fast_icp.hpp +++ b/modules/rgbd/src/fast_icp.hpp @@ -22,7 +22,6 @@ class ICP InputArray oldPoints, InputArray oldNormals, InputArray newPoints, InputArray newNormals ) const = 0; - virtual ~ICP() { } protected: diff --git a/modules/rgbd/src/hash_tsdf.cpp b/modules/rgbd/src/hash_tsdf.cpp index b414500217f..35838352d43 100644 --- a/modules/rgbd/src/hash_tsdf.cpp +++ b/modules/rgbd/src/hash_tsdf.cpp @@ -17,55 +17,63 @@ #include "utils.hpp" #define USE_INTERPOLATION_IN_GETNORMAL 1 - +#define VOLUMES_SIZE 1024 namespace cv { namespace kinfu { -static inline TsdfType floatToTsdf(float num) -{ - //CV_Assert(-1 < num <= 1); - int8_t res = int8_t(num * (-128.f)); - res = res ? res : (num < 0 ? 1 : -1); - return res; -} - -static inline float tsdfToFloat(TsdfType num) -{ - return float(num) * (-1.f / 128.f); -} - HashTSDFVolume::HashTSDFVolume(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, - float _truncDist, int _maxWeight, float _truncateThreshold, - int _volumeUnitRes, bool _zFirstMemOrder) + float _truncDist, int _maxWeight, float _truncateThreshold, + int _volumeUnitRes, bool _zFirstMemOrder) : Volume(_voxelSize, _pose, _raycastStepFactor), - maxWeight(_maxWeight), - truncateThreshold(_truncateThreshold), - volumeUnitResolution(_volumeUnitRes), - volumeUnitSize(voxelSize * volumeUnitResolution), - zFirstMemOrder(_zFirstMemOrder) + maxWeight(_maxWeight), + truncateThreshold(_truncateThreshold), + volumeUnitResolution(_volumeUnitRes), + volumeUnitSize(voxelSize* volumeUnitResolution), + zFirstMemOrder(_zFirstMemOrder) { truncDist = std::max(_truncDist, 4.0f * voxelSize); } -HashTSDFVolumeCPU::HashTSDFVolumeCPU(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, - float _truncDist, int _maxWeight, float _truncateThreshold, - int _volumeUnitRes, bool _zFirstMemOrder) - : HashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, - _truncateThreshold, _volumeUnitRes, _zFirstMemOrder) +HashTSDFVolumeCPU::HashTSDFVolumeCPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, + int _maxWeight, float _truncateThreshold, int _volumeUnitRes, bool _zFirstMemOrder) + :HashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold, _volumeUnitRes, + _zFirstMemOrder) { + int xdim, ydim, zdim; + if (zFirstMemOrder) + { + xdim = volumeUnitResolution * volumeUnitResolution; + ydim = volumeUnitResolution; + zdim = 1; + } + else + { + xdim = 1; + ydim = volumeUnitResolution; + zdim = volumeUnitResolution * volumeUnitResolution; + } + volStrides = Vec4i(xdim, ydim, zdim); + + reset(); } +HashTSDFVolumeCPU::HashTSDFVolumeCPU(const VolumeParams& _params, bool _zFirstMemOrder) + : HashTSDFVolume(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, _params.tsdfTruncDist, _params.maxWeight, + _params.depthTruncThreshold, _params.unitResolution, _zFirstMemOrder) +{ +} // zero volume, leave rest params the same void HashTSDFVolumeCPU::reset() { CV_TRACE_FUNCTION(); - volumeUnits.clear(); + lastVolIndex = 0; + volUnitsData = cv::Mat(VOLUMES_SIZE, volumeUnitResolution * volumeUnitResolution * volumeUnitResolution, rawType()); } -void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics) +void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics, const int frameId) { CV_TRACE_FUNCTION(); @@ -73,7 +81,7 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma Depth depth = _depth.getMat(); //! Compute volumes to be allocated - const int depthStride = 1; + const int depthStride = int(log2(volumeUnitResolution)); const float invDepthFactor = 1.f / depthFactor; const Intr::Reprojector reproj(intrinsics.makeReprojector()); const Affine3f cam2vol(pose.inv() * Affine3f(cameraPose)); @@ -81,6 +89,7 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma VolumeUnitIndexSet newIndices; Mutex mutex; Range allocateRange(0, depth.rows); + auto AllocateVolumeUnitsInvoker = [&](const Range& range) { VolumeUnitIndexSet localAccessVolUnits; for (int y = range.start; y < range.end; y += depthStride) @@ -127,14 +136,24 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma parallel_for_(allocateRange, AllocateVolumeUnitsInvoker); //! Perform the allocation - int res = volumeUnitResolution; - Point3i volumeDims(res, res, res); for (auto idx : newIndices) { VolumeUnit& vu = volumeUnits[idx]; Matx44f subvolumePose = pose.translate(volumeUnitIdxToVolume(idx)).matrix; - vu.pVolume = makePtr(voxelSize, subvolumePose, raycastStepFactor, truncDist, maxWeight, volumeDims); + + vu.pose = subvolumePose; + vu.index = lastVolIndex; lastVolIndex++; + if (lastVolIndex > VolumeIndex(volUnitsData.size().height)) + { + volUnitsData.resize((lastVolIndex - 1) * 2); + } + volUnitsData.row(vu.index).forEach([](VecTsdfVoxel& vv, const int* /* position */) + { + TsdfVoxel& v = reinterpret_cast(vv); + v.tsdf = floatToTsdf(0.0f); v.weight = 0; + }); //! This volume unit will definitely be required for current integration + vu.lastVisibleIndex = frameId; vu.isActive = true; } @@ -154,7 +173,7 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma for (int i = range.start; i < range.end; ++i) { Vec3i tsdf_idx = totalVolUnits[i]; - VolumeUnitMap::iterator it = volumeUnits.find(tsdf_idx); + VolumeUnitIndexes::iterator it = volumeUnits.find(tsdf_idx); if (it == volumeUnits.end()) return; @@ -169,17 +188,27 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma if (cameraPoint.x >= 0 && cameraPoint.y >= 0 && cameraPoint.x < depth.cols && cameraPoint.y < depth.rows) { assert(it != volumeUnits.end()); - it->second.isActive = true; + it->second.lastVisibleIndex = frameId; + it->second.isActive = true; } } }); + Vec6f newParams((float)depth.rows, (float)depth.cols, + intrinsics.fx, intrinsics.fy, + intrinsics.cx, intrinsics.cy); + if ( !(frameParams==newParams) ) + { + frameParams = newParams; + pixNorms = preCalculationPixNorm(depth, intrinsics); + } + //! Integrate the correct volumeUnits parallel_for_(Range(0, (int)totalVolUnits.size()), [&](const Range& range) { for (int i = range.start; i < range.end; i++) { Vec3i tsdf_idx = totalVolUnits[i]; - VolumeUnitMap::iterator it = volumeUnits.find(tsdf_idx); + VolumeUnitIndexes::iterator it = volumeUnits.find(tsdf_idx); if (it == volumeUnits.end()) return; @@ -187,7 +216,10 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma if (volumeUnit.isActive) { //! The volume unit should already be added into the Volume from the allocator - volumeUnit.pVolume->integrate(depth, depthFactor, cameraPose, intrinsics); + integrateVolumeUnit(truncDist, voxelSize, maxWeight, volumeUnit.pose, + Point3i(volumeUnitResolution, volumeUnitResolution, volumeUnitResolution), volStrides, depth, + depthFactor, cameraPose, intrinsics, pixNorms, volUnitsData.row(volumeUnit.index)); + //! Ensure all active volumeUnits are set to inactive for next integration volumeUnit.isActive = false; } @@ -195,45 +227,64 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma }); } -cv::Vec3i HashTSDFVolumeCPU::volumeToVolumeUnitIdx(cv::Point3f p) const +cv::Vec3i HashTSDFVolumeCPU::volumeToVolumeUnitIdx(const cv::Point3f& p) const { return cv::Vec3i(cvFloor(p.x / volumeUnitSize), cvFloor(p.y / volumeUnitSize), cvFloor(p.z / volumeUnitSize)); } -cv::Point3f HashTSDFVolumeCPU::volumeUnitIdxToVolume(cv::Vec3i volumeUnitIdx) const +cv::Point3f HashTSDFVolumeCPU::volumeUnitIdxToVolume(const cv::Vec3i& volumeUnitIdx) const { return cv::Point3f(volumeUnitIdx[0] * volumeUnitSize, volumeUnitIdx[1] * volumeUnitSize, volumeUnitIdx[2] * volumeUnitSize); } -cv::Point3f HashTSDFVolumeCPU::voxelCoordToVolume(cv::Vec3i voxelIdx) const +cv::Point3f HashTSDFVolumeCPU::voxelCoordToVolume(const cv::Vec3i& voxelIdx) const { return cv::Point3f(voxelIdx[0] * voxelSize, voxelIdx[1] * voxelSize, voxelIdx[2] * voxelSize); } -cv::Vec3i HashTSDFVolumeCPU::volumeToVoxelCoord(cv::Point3f point) const +cv::Vec3i HashTSDFVolumeCPU::volumeToVoxelCoord(const cv::Point3f& point) const { return cv::Vec3i(cvFloor(point.x * voxelSizeInv), cvFloor(point.y * voxelSizeInv), cvFloor(point.z * voxelSizeInv)); } +inline TsdfVoxel HashTSDFVolumeCPU::_at(const cv::Vec3i& volumeIdx, VolumeIndex indx) const +{ + //! Out of bounds + if ((volumeIdx[0] >= volumeUnitResolution || volumeIdx[0] < 0) || + (volumeIdx[1] >= volumeUnitResolution || volumeIdx[1] < 0) || + (volumeIdx[2] >= volumeUnitResolution || volumeIdx[2] < 0)) + { + TsdfVoxel dummy; + dummy.tsdf = floatToTsdf(1.0f); + dummy.weight = 0; + return dummy; + } + + const TsdfVoxel* volData = volUnitsData.ptr(indx); + int coordBase = + volumeIdx[0] * volStrides[0] + volumeIdx[1] * volStrides[1] + volumeIdx[2] * volStrides[2]; + return volData[coordBase]; +} + inline TsdfVoxel HashTSDFVolumeCPU::at(const cv::Vec3i& volumeIdx) const + { - cv::Vec3i volumeUnitIdx = cv::Vec3i(cvFloor(volumeIdx[0] / volumeUnitResolution), - cvFloor(volumeIdx[1] / volumeUnitResolution), - cvFloor(volumeIdx[2] / volumeUnitResolution)); + Vec3i volumeUnitIdx = Vec3i(cvFloor(volumeIdx[0] / volumeUnitResolution), + cvFloor(volumeIdx[1] / volumeUnitResolution), + cvFloor(volumeIdx[2] / volumeUnitResolution)); + + VolumeUnitIndexes::const_iterator it = volumeUnits.find(volumeUnitIdx); - VolumeUnitMap::const_iterator it = volumeUnits.find(volumeUnitIdx); if (it == volumeUnits.end()) { TsdfVoxel dummy; - dummy.tsdf = floatToTsdf(1.f); + dummy.tsdf = floatToTsdf(1.f); dummy.weight = 0; return dummy; } - cv::Ptr volumeUnit = - std::dynamic_pointer_cast(it->second.pVolume); cv::Vec3i volUnitLocalIdx = volumeIdx - cv::Vec3i(volumeUnitIdx[0] * volumeUnitResolution, volumeUnitIdx[1] * volumeUnitResolution, @@ -241,28 +292,28 @@ inline TsdfVoxel HashTSDFVolumeCPU::at(const cv::Vec3i& volumeIdx) const volUnitLocalIdx = cv::Vec3i(abs(volUnitLocalIdx[0]), abs(volUnitLocalIdx[1]), abs(volUnitLocalIdx[2])); - return volumeUnit->at(volUnitLocalIdx); + return _at(volUnitLocalIdx, it->second.index); + } -inline TsdfVoxel HashTSDFVolumeCPU::at(const cv::Point3f& point) const +TsdfVoxel HashTSDFVolumeCPU::at(const Point3f& point) const { cv::Vec3i volumeUnitIdx = volumeToVolumeUnitIdx(point); - VolumeUnitMap::const_iterator it = volumeUnits.find(volumeUnitIdx); + VolumeUnitIndexes::const_iterator it = volumeUnits.find(volumeUnitIdx); + if (it == volumeUnits.end()) { TsdfVoxel dummy; - dummy.tsdf = floatToTsdf(1.f); + dummy.tsdf = floatToTsdf(1.f); dummy.weight = 0; return dummy; } - cv::Ptr volumeUnit = - std::dynamic_pointer_cast(it->second.pVolume); cv::Point3f volumeUnitPos = volumeUnitIdxToVolume(volumeUnitIdx); cv::Vec3i volUnitLocalIdx = volumeToVoxelCoord(point - volumeUnitPos); volUnitLocalIdx = cv::Vec3i(abs(volUnitLocalIdx[0]), abs(volUnitLocalIdx[1]), abs(volUnitLocalIdx[2])); - return volumeUnit->at(volUnitLocalIdx); + return _at(volUnitLocalIdx, it->second.index); } static inline Vec3i voxelToVolumeUnitIdx(const Vec3i& pt, const int vuRes) @@ -281,27 +332,25 @@ static inline Vec3i voxelToVolumeUnitIdx(const Vec3i& pt, const int vuRes) } } -inline TsdfVoxel atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, VolumeUnitMap::const_iterator it, - VolumeUnitMap::const_iterator vend, int unitRes) +TsdfVoxel HashTSDFVolumeCPU::atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, VolumeUnitIndexes::const_iterator it) const { - if (it == vend) + if (it == volumeUnits.end()) { TsdfVoxel dummy; dummy.tsdf = floatToTsdf(1.f); dummy.weight = 0; return dummy; } - Ptr volumeUnit = std::dynamic_pointer_cast(it->second.pVolume); - - Vec3i volUnitLocalIdx = point - volumeUnitIdx * unitRes; + Vec3i volUnitLocalIdx = point - volumeUnitIdx * volumeUnitResolution; // expanding at(), removing bounds check - const TsdfVoxel* volData = volumeUnit->volume.ptr(); - Vec4i volDims = volumeUnit->volDims; - int coordBase = volUnitLocalIdx[0] * volDims[0] + volUnitLocalIdx[1] * volDims[1] + volUnitLocalIdx[2] * volDims[2]; + const TsdfVoxel* volData = volUnitsData.ptr(it->second.index); + int coordBase = volUnitLocalIdx[0] * volStrides[0] + volUnitLocalIdx[1] * volStrides[1] + volUnitLocalIdx[2] * volStrides[2]; return volData[coordBase]; } + + #if USE_INTRINSICS inline float interpolate(float tx, float ty, float tz, float vx[8]) { @@ -343,7 +392,7 @@ float HashTSDFVolumeCPU::interpolateVoxelPoint(const Point3f& point) const // A small hash table to reduce a number of find() calls bool queried[8]; - VolumeUnitMap::const_iterator iterMap[8]; + VolumeUnitIndexes::const_iterator iterMap[8]; for (int i = 0; i < 8; i++) { iterMap[i] = volumeUnits.end(); @@ -373,9 +422,8 @@ float HashTSDFVolumeCPU::interpolateVoxelPoint(const Point3f& point) const iterMap[dictIdx] = it; queried[dictIdx] = true; } - //VolumeUnitMap::const_iterator it = volumeUnits.find(volumeUnitIdx); - vx[i] = atVolumeUnit(pt, volumeUnitIdx, it, volumeUnits.end(), volumeUnitResolution).tsdf; + vx[i] = atVolumeUnit(pt, volumeUnitIdx, it).tsdf; } return interpolate(tx, ty, tz, vx); @@ -386,7 +434,8 @@ inline float HashTSDFVolumeCPU::interpolateVoxel(const cv::Point3f& point) const return interpolateVoxelPoint(point * voxelSizeInv); } -inline Point3f HashTSDFVolumeCPU::getNormalVoxel(Point3f point) const + +Point3f HashTSDFVolumeCPU::getNormalVoxel(const Point3f &point) const { Vec3f normal = Vec3f(0, 0, 0); @@ -395,7 +444,7 @@ inline Point3f HashTSDFVolumeCPU::getNormalVoxel(Point3f point) const // A small hash table to reduce a number of find() calls bool queried[8]; - VolumeUnitMap::const_iterator iterMap[8]; + VolumeUnitIndexes::const_iterator iterMap[8]; for (int i = 0; i < 8; i++) { iterMap[i] = volumeUnits.end(); @@ -436,9 +485,8 @@ inline Point3f HashTSDFVolumeCPU::getNormalVoxel(Point3f point) const iterMap[dictIdx] = it; queried[dictIdx] = true; } - //VolumeUnitMap::const_iterator it = volumeUnits.find(volumeUnitIdx); - vals[i] = tsdfToFloat(atVolumeUnit(pt, volumeUnitIdx, it, volumeUnits.end(), volumeUnitResolution).tsdf); + vals[i] = tsdfToFloat(atVolumeUnit(pt, volumeUnitIdx, it).tsdf); } #if !USE_INTERPOLATION_IN_GETNORMAL @@ -525,26 +573,33 @@ inline Point3f HashTSDFVolumeCPU::getNormalVoxel(Point3f point) const return nv < 0.0001f ? nan3 : normal / nv; } -struct HashRaycastInvoker : ParallelLoopBody +void HashTSDFVolumeCPU::raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, + OutputArray _points, OutputArray _normals) const { - HashRaycastInvoker(Points& _points, Normals& _normals, const Matx44f& cameraPose, - const Intr& intrinsics, const HashTSDFVolumeCPU& _volume) - : ParallelLoopBody(), - points(_points), - normals(_normals), - volume(_volume), - tstep(_volume.truncDist * _volume.raycastStepFactor), - cam2vol(volume.pose.inv() * Affine3f(cameraPose)), - vol2cam(Affine3f(cameraPose.inv()) * volume.pose), - reproj(intrinsics.makeReprojector()) - { - } + CV_TRACE_FUNCTION(); + CV_Assert(frameSize.area() > 0); + + _points.create(frameSize, POINT_TYPE); + _normals.create(frameSize, POINT_TYPE); + + Points points1 = _points.getMat(); + Normals normals1 = _normals.getMat(); + + Points& points(points1); + Normals& normals(normals1); + const HashTSDFVolumeCPU& volume(*this); + const float tstep(volume.truncDist * volume.raycastStepFactor); + const Affine3f cam2vol(volume.pose.inv() * Affine3f(cameraPose)); + const Affine3f vol2cam(Affine3f(cameraPose.inv()) * volume.pose); + const Intr::Reprojector reproj(intrinsics.makeReprojector()); + + const int nstripes = -1; - virtual void operator()(const Range& range) const override + auto _HashRaycastInvoker = [&](const Range& range) { const Point3f cam2volTrans = cam2vol.translation(); - const Matx33f cam2volRot = cam2vol.rotation(); - const Matx33f vol2camRot = vol2cam.rotation(); + const Matx33f cam2volRot = cam2vol.rotation(); + const Matx33f vol2camRot = vol2cam.rotation(); const float blockSize = volume.volumeUnitSize; @@ -559,53 +614,53 @@ struct HashRaycastInvoker : ParallelLoopBody Point3f point = nan3, normal = nan3; //! Ray origin and direction in the volume coordinate frame - Point3f orig = cam2volTrans; - Point3f rayDirV = - normalize(Vec3f(cam2volRot * reproj(Point3f(float(x), float(y), 1.f)))); + Point3f orig = cam2volTrans; + Point3f rayDirV = normalize(Vec3f(cam2volRot * reproj(Point3f(float(x), float(y), 1.f)))); - float tmin = 0; - float tmax = volume.truncateThreshold; + float tmin = 0; + float tmax = volume.truncateThreshold; float tcurr = tmin; cv::Vec3i prevVolumeUnitIdx = cv::Vec3i(std::numeric_limits::min(), std::numeric_limits::min(), - std::numeric_limits::min()); + std::numeric_limits::min()); - float tprev = tcurr; + + float tprev = tcurr; float prevTsdf = volume.truncDist; - cv::Ptr currVolumeUnit; + Ptr currVolumeUnit; while (tcurr < tmax) { - Point3f currRayPos = orig + tcurr * rayDirV; + Point3f currRayPos = orig + tcurr * rayDirV; cv::Vec3i currVolumeUnitIdx = volume.volumeToVolumeUnitIdx(currRayPos); - VolumeUnitMap::const_iterator it = volume.volumeUnits.find(currVolumeUnitIdx); + + VolumeUnitIndexes::const_iterator it = volume.volumeUnits.find(currVolumeUnitIdx); float currTsdf = prevTsdf; - int currWeight = 0; - float stepSize = 0.5f * blockSize; + int currWeight = 0; + float stepSize = 0.5f * blockSize; cv::Vec3i volUnitLocalIdx; - //! Does the subvolume exist in hashtable + + //! The subvolume exists in hashtable if (it != volume.volumeUnits.end()) { - currVolumeUnit = - std::dynamic_pointer_cast(it->second.pVolume); cv::Point3f currVolUnitPos = volume.volumeUnitIdxToVolume(currVolumeUnitIdx); volUnitLocalIdx = volume.volumeToVoxelCoord(currRayPos - currVolUnitPos); + //! TODO: Figure out voxel interpolation - TsdfVoxel currVoxel = currVolumeUnit->at(volUnitLocalIdx); - currTsdf = tsdfToFloat(currVoxel.tsdf); - currWeight = currVoxel.weight; - stepSize = tstep; + TsdfVoxel currVoxel = _at(volUnitLocalIdx, it->second.index); + currTsdf = tsdfToFloat(currVoxel.tsdf); + currWeight = currVoxel.weight; + stepSize = tstep; } //! Surface crossing if (prevTsdf > 0.f && currTsdf <= 0.f && currWeight > 0) { - float tInterp = - (tcurr * prevTsdf - tprev * currTsdf) / (prevTsdf - currTsdf); + float tInterp = (tcurr * prevTsdf - tprev * currTsdf) / (prevTsdf - currTsdf); if (!cvIsNaN(tInterp) && !cvIsInf(tInterp)) { Point3f pv = orig + tInterp * rayDirV; @@ -614,132 +669,92 @@ struct HashRaycastInvoker : ParallelLoopBody if (!isNaN(nv)) { normal = vol2camRot * nv; - point = vol2cam * pv; + point = vol2cam * pv; } } break; } prevVolumeUnitIdx = currVolumeUnitIdx; - prevTsdf = currTsdf; - tprev = tcurr; + prevTsdf = currTsdf; + tprev = tcurr; tcurr += stepSize; } ptsRow[x] = toPtype(point); nrmRow[x] = toPtype(normal); } } - } + }; + + parallel_for_(Range(0, points.rows), _HashRaycastInvoker, nstripes); +} - Points& points; - Normals& normals; - const HashTSDFVolumeCPU& volume; - const float tstep; - const Affine3f cam2vol; - const Affine3f vol2cam; - const Intr::Reprojector reproj; -}; - -void HashTSDFVolumeCPU::raycast(const cv::Matx44f& cameraPose, const cv::kinfu::Intr& intrinsics, - cv::Size frameSize, cv::OutputArray _points, - cv::OutputArray _normals) const +void HashTSDFVolumeCPU::fetchPointsNormals(OutputArray _points, OutputArray _normals) const { CV_TRACE_FUNCTION(); - CV_Assert(frameSize.area() > 0); - _points.create(frameSize, POINT_TYPE); - _normals.create(frameSize, POINT_TYPE); - - Points points = _points.getMat(); - Normals normals = _normals.getMat(); + if (_points.needed()) + { + std::vector> pVecs, nVecs; - HashRaycastInvoker ri(points, normals, cameraPose, intrinsics, *this); + std::vector totalVolUnits; + for (const auto& keyvalue : volumeUnits) + { + totalVolUnits.push_back(keyvalue.first); + } + Range fetchRange(0, (int)totalVolUnits.size()); + const int nstripes = -1; - const int nstripes = -1; - parallel_for_(Range(0, points.rows), ri, nstripes); -} + const HashTSDFVolumeCPU& volume(*this); + bool needNormals(_normals.needed()); + Mutex mutex; -struct HashFetchPointsNormalsInvoker : ParallelLoopBody -{ - HashFetchPointsNormalsInvoker(const HashTSDFVolumeCPU& _volume, - const std::vector& _totalVolUnits, - std::vector>& _pVecs, - std::vector>& _nVecs, bool _needNormals) - : ParallelLoopBody(), - volume(_volume), - totalVolUnits(_totalVolUnits), - pVecs(_pVecs), - nVecs(_nVecs), - needNormals(_needNormals) - { - } - virtual void operator()(const Range& range) const override - { - std::vector points, normals; - for (int i = range.start; i < range.end; i++) + auto HashFetchPointsNormalsInvoker = [&](const Range& range) { - cv::Vec3i tsdf_idx = totalVolUnits[i]; - VolumeUnitMap::const_iterator it = volume.volumeUnits.find(tsdf_idx); - Point3f base_point = volume.volumeUnitIdxToVolume(tsdf_idx); - if (it != volume.volumeUnits.end()) + + std::vector points, normals; + for (int i = range.start; i < range.end; i++) { - cv::Ptr volumeUnit = - std::dynamic_pointer_cast(it->second.pVolume); - std::vector localPoints; - std::vector localNormals; - for (int x = 0; x < volume.volumeUnitResolution; x++) - for (int y = 0; y < volume.volumeUnitResolution; y++) - for (int z = 0; z < volume.volumeUnitResolution; z++) - { - cv::Vec3i voxelIdx(x, y, z); - TsdfVoxel voxel = volumeUnit->at(voxelIdx); + cv::Vec3i tsdf_idx = totalVolUnits[i]; - if (voxel.tsdf != -128 && voxel.weight != 0) + + VolumeUnitIndexes::const_iterator it = volume.volumeUnits.find(tsdf_idx); + Point3f base_point = volume.volumeUnitIdxToVolume(tsdf_idx); + if (it != volume.volumeUnits.end()) + { + std::vector localPoints; + std::vector localNormals; + for (int x = 0; x < volume.volumeUnitResolution; x++) + for (int y = 0; y < volume.volumeUnitResolution; y++) + for (int z = 0; z < volume.volumeUnitResolution; z++) { - Point3f point = base_point + volume.voxelCoordToVolume(voxelIdx); - localPoints.push_back(toPtype(point)); - if (needNormals) + cv::Vec3i voxelIdx(x, y, z); + TsdfVoxel voxel = _at(voxelIdx, it->second.index); + + if (voxel.tsdf != -128 && voxel.weight != 0) { - Point3f normal = volume.getNormalVoxel(point); - localNormals.push_back(toPtype(normal)); + Point3f point = base_point + volume.voxelCoordToVolume(voxelIdx); + localPoints.push_back(toPtype(point)); + if (needNormals) + { + Point3f normal = volume.getNormalVoxel(point); + localNormals.push_back(toPtype(normal)); + } } } - } - AutoLock al(mutex); - pVecs.push_back(localPoints); - nVecs.push_back(localNormals); + AutoLock al(mutex); + pVecs.push_back(localPoints); + nVecs.push_back(localNormals); + } } - } - } + }; - const HashTSDFVolumeCPU& volume; - std::vector totalVolUnits; - std::vector>& pVecs; - std::vector>& nVecs; - const TsdfVoxel* volDataStart; - bool needNormals; - mutable Mutex mutex; -}; + parallel_for_(fetchRange, HashFetchPointsNormalsInvoker, nstripes); -void HashTSDFVolumeCPU::fetchPointsNormals(OutputArray _points, OutputArray _normals) const -{ - CV_TRACE_FUNCTION(); - if (_points.needed()) - { - std::vector> pVecs, nVecs; - std::vector totalVolUnits; - for (const auto& keyvalue : volumeUnits) - { - totalVolUnits.push_back(keyvalue.first); - } - HashFetchPointsNormalsInvoker fi(*this, totalVolUnits, pVecs, nVecs, _normals.needed()); - Range range(0, (int)totalVolUnits.size()); - const int nstripes = -1; - parallel_for_(range, fi, nstripes); std::vector points, normals; for (size_t i = 0; i < pVecs.size(); i++) { @@ -760,7 +775,7 @@ void HashTSDFVolumeCPU::fetchPointsNormals(OutputArray _points, OutputArray _nor } } -void HashTSDFVolumeCPU::fetchNormals(cv::InputArray _points, cv::OutputArray _normals) const +void HashTSDFVolumeCPU::fetchNormals(InputArray _points, OutputArray _normals) const { CV_TRACE_FUNCTION(); @@ -773,8 +788,7 @@ void HashTSDFVolumeCPU::fetchNormals(cv::InputArray _points, cv::OutputArray _no Normals normals = _normals.getMat(); const HashTSDFVolumeCPU& _volume = *this; - auto HashPushNormals = [&](const ptype& point, const int* position) - { + auto HashPushNormals = [&](const ptype& point, const int* position) { const HashTSDFVolumeCPU& volume(_volume); Affine3f invPose(volume.pose.inv()); Point3f p = fromPtype(point); @@ -782,7 +796,7 @@ void HashTSDFVolumeCPU::fetchNormals(cv::InputArray _points, cv::OutputArray _no if (!isNaN(p)) { Point3f voxelPoint = invPose * p; - n = volume.pose.rotation() * volume.getNormalVoxel(voxelPoint); + n = volume.pose.rotation() * volume.getNormalVoxel(voxelPoint); } normals(position[0], position[1]) = toPtype(n); }; @@ -790,13 +804,17 @@ void HashTSDFVolumeCPU::fetchNormals(cv::InputArray _points, cv::OutputArray _no } } -cv::Ptr makeHashTSDFVolume(float _voxelSize, cv::Matx44f _pose, - float _raycastStepFactor, float _truncDist, - int _maxWeight, float _truncateThreshold, - int _volumeUnitResolution) +int HashTSDFVolumeCPU::getVisibleBlocks(int currFrameId, int frameThreshold) const { - return cv::makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, - _maxWeight, _truncateThreshold, _volumeUnitResolution); + int numVisibleBlocks = 0; + //! TODO: Iterate over map parallely? + for (const auto& keyvalue : volumeUnits) + { + const VolumeUnit& volumeUnit = keyvalue.second; + if (volumeUnit.lastVisibleIndex > (currFrameId - frameThreshold)) + numVisibleBlocks++; + } + return numVisibleBlocks; } } // namespace kinfu diff --git a/modules/rgbd/src/hash_tsdf.hpp b/modules/rgbd/src/hash_tsdf.hpp index bac5ea9a46a..31bf026785b 100644 --- a/modules/rgbd/src/hash_tsdf.hpp +++ b/modules/rgbd/src/hash_tsdf.hpp @@ -9,7 +9,7 @@ #include #include -#include "tsdf.hpp" +#include "tsdf_functions.hpp" namespace cv { @@ -35,20 +35,10 @@ class HashTSDFVolume : public Volume bool zFirstMemOrder; }; -struct VolumeUnit -{ - VolumeUnit() : pVolume(nullptr){}; - ~VolumeUnit() = default; - - cv::Ptr pVolume; - cv::Vec3i index; - bool isActive; -}; - //! Spatial hashing struct tsdf_hash { - size_t operator()(const cv::Vec3i& x) const noexcept + size_t operator()(const Vec3i& x) const noexcept { size_t seed = 0; constexpr uint32_t GOLDEN_RATIO = 0x9e3779b9; @@ -60,54 +50,84 @@ struct tsdf_hash } }; +typedef unsigned int VolumeIndex; +struct VolumeUnit +{ + cv::Vec3i coord; + VolumeIndex index; + cv::Matx44f pose; + int lastVisibleIndex = 0; + bool isActive; +}; + typedef std::unordered_set VolumeUnitIndexSet; -typedef std::unordered_map VolumeUnitMap; +typedef std::unordered_map VolumeUnitIndexes; class HashTSDFVolumeCPU : public HashTSDFVolume { public: // dimension in voxels, size in meters - HashTSDFVolumeCPU(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, - float _truncDist, int _maxWeight, float _truncateThreshold, - int _volumeUnitRes, bool zFirstMemOrder = true); + HashTSDFVolumeCPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, + float _truncateThreshold, int _volumeUnitRes, bool zFirstMemOrder = true); + + HashTSDFVolumeCPU(const VolumeParams& _volumeParams, bool zFirstMemOrder = true); - virtual void integrate(InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, - const cv::kinfu::Intr& intrinsics) override; - virtual void raycast(const cv::Matx44f& cameraPose, const cv::kinfu::Intr& intrinsics, - cv::Size frameSize, cv::OutputArray points, - cv::OutputArray normals) const override; + void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const kinfu::Intr& intrinsics, + const int frameId = 0) override; + void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, + OutputArray normals) const override; - virtual void fetchNormals(cv::InputArray points, cv::OutputArray _normals) const override; - virtual void fetchPointsNormals(cv::OutputArray points, cv::OutputArray normals) const override; + void fetchNormals(InputArray points, OutputArray _normals) const override; + void fetchPointsNormals(OutputArray points, OutputArray normals) const override; - virtual void reset() override; + void reset() override; + size_t getTotalVolumeUnits() const { return volumeUnits.size(); } + int getVisibleBlocks(int currFrameId, int frameThreshold) const; //! Return the voxel given the voxel index in the universal volume (1 unit = 1 voxel_length) - virtual TsdfVoxel at(const cv::Vec3i& volumeIdx) const; + TsdfVoxel at(const Vec3i& volumeIdx) const; //! Return the voxel given the point in volume coordinate system i.e., (metric scale 1 unit = //! 1m) virtual TsdfVoxel at(const cv::Point3f& point) const; + virtual TsdfVoxel _at(const cv::Vec3i& volumeIdx, VolumeIndex indx) const; + + TsdfVoxel atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, VolumeUnitIndexes::const_iterator it) const; + float interpolateVoxelPoint(const Point3f& point) const; - inline float interpolateVoxel(const cv::Point3f& point) const; - Point3f getNormalVoxel(cv::Point3f p) const; + float interpolateVoxel(const cv::Point3f& point) const; + Point3f getNormalVoxel(const cv::Point3f& p) const; //! Utility functions for coordinate transformations - cv::Vec3i volumeToVolumeUnitIdx(cv::Point3f point) const; - cv::Point3f volumeUnitIdxToVolume(cv::Vec3i volumeUnitIdx) const; + Vec3i volumeToVolumeUnitIdx(const Point3f& point) const; + Point3f volumeUnitIdxToVolume(const Vec3i& volumeUnitIdx) const; - cv::Point3f voxelCoordToVolume(cv::Vec3i voxelIdx) const; - cv::Vec3i volumeToVoxelCoord(cv::Point3f point) const; + Point3f voxelCoordToVolume(const Vec3i& voxelIdx) const; + Vec3i volumeToVoxelCoord(const Point3f& point) const; public: - //! Hashtable of individual smaller volume units - VolumeUnitMap volumeUnits; + Vec4i volStrides; + Vec6f frameParams; + Mat pixNorms; + VolumeUnitIndexes volumeUnits; + cv::Mat volUnitsData; + VolumeIndex lastVolIndex; }; -cv::Ptr makeHashTSDFVolume(float _voxelSize, cv::Matx44f _pose, - float _raycastStepFactor, float _truncDist, - int _maxWeight, float truncateThreshold, - int volumeUnitResolution = 16); + +template +Ptr makeHashTSDFVolume(const VolumeParams& _volumeParams) +{ + return makePtr(_volumeParams); +} + +template +Ptr makeHashTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, + int _maxWeight, float truncateThreshold, int volumeUnitResolution = 16) +{ + return makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, truncateThreshold, + volumeUnitResolution); +} } // namespace kinfu } // namespace cv #endif diff --git a/modules/rgbd/src/kinfu.cpp b/modules/rgbd/src/kinfu.cpp index d776f359ec3..1d8314aae6f 100644 --- a/modules/rgbd/src/kinfu.cpp +++ b/modules/rgbd/src/kinfu.cpp @@ -119,7 +119,6 @@ class KinFuImpl : public KinFu void render(OutputArray image, const Matx44f& cameraPose) const CV_OVERRIDE; - //! TODO(Akash): Add back later virtual void getCloud(OutputArray points, OutputArray normals) const CV_OVERRIDE; void getPoints(OutputArray points) const CV_OVERRIDE; void getNormals(InputArray points, OutputArray normals) const CV_OVERRIDE; diff --git a/modules/rgbd/src/large_kinfu.cpp b/modules/rgbd/src/large_kinfu.cpp new file mode 100644 index 00000000000..dedeabb82db --- /dev/null +++ b/modules/rgbd/src/large_kinfu.cpp @@ -0,0 +1,361 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this +// module's directory + +#include "fast_icp.hpp" +#include "hash_tsdf.hpp" +#include "kinfu_frame.hpp" +#include "precomp.hpp" +#include "submap.hpp" +#include "tsdf.hpp" + +namespace cv +{ +namespace large_kinfu +{ +using namespace kinfu; + +Ptr Params::defaultParams() +{ + Params p; + + //! Frame parameters + { + p.frameSize = Size(640, 480); + + float fx, fy, cx, cy; + fx = fy = 525.f; + cx = p.frameSize.width / 2.0f - 0.5f; + cy = p.frameSize.height / 2.0f - 0.5f; + p.intr = Matx33f(fx, 0, cx, 0, fy, cy, 0, 0, 1); + + // 5000 for the 16-bit PNG files + // 1 for the 32-bit float images in the ROS bag files + p.depthFactor = 5000; + + // sigma_depth is scaled by depthFactor when calling bilateral filter + p.bilateral_sigma_depth = 0.04f; // meter + p.bilateral_sigma_spatial = 4.5; // pixels + p.bilateral_kernel_size = 7; // pixels + p.truncateThreshold = 0.f; // meters + } + //! ICP parameters + { + p.icpAngleThresh = (float)(30. * CV_PI / 180.); // radians + p.icpDistThresh = 0.1f; // meters + + p.icpIterations = { 10, 5, 4 }; + p.pyramidLevels = (int)p.icpIterations.size(); + } + //! Volume parameters + { + float volumeSize = 3.0f; + p.volumeParams.type = VolumeType::TSDF; + p.volumeParams.resolution = Vec3i::all(512); + p.volumeParams.pose = Affine3f().translate(Vec3f(-volumeSize / 2.f, -volumeSize / 2.f, 0.5f)); + p.volumeParams.voxelSize = volumeSize / 512.f; // meters + p.volumeParams.tsdfTruncDist = 7 * p.volumeParams.voxelSize; // about 0.04f in meters + p.volumeParams.maxWeight = 64; // frames + p.volumeParams.raycastStepFactor = 0.25f; // in voxel sizes + p.volumeParams.depthTruncThreshold = p.truncateThreshold; + } + //! Unused parameters + p.tsdf_min_camera_movement = 0.f; // meters, disabled + p.lightPose = Vec3f::all(0.f); // meters + + return makePtr(p); +} + +Ptr Params::coarseParams() +{ + Ptr p = defaultParams(); + + //! Reduce ICP iterations and pyramid levels + { + p->icpIterations = { 5, 3, 2 }; + p->pyramidLevels = (int)p->icpIterations.size(); + } + //! Make the volume coarse + { + float volumeSize = 3.f; + p->volumeParams.resolution = Vec3i::all(128); // number of voxels + p->volumeParams.voxelSize = volumeSize / 128.f; + p->volumeParams.tsdfTruncDist = 2 * p->volumeParams.voxelSize; // 0.04f in meters + p->volumeParams.raycastStepFactor = 0.75f; // in voxel sizes + } + return p; +} +Ptr Params::hashTSDFParams(bool isCoarse) +{ + Ptr p; + if (isCoarse) + p = coarseParams(); + else + p = defaultParams(); + + p->volumeParams.type = VolumeType::HASHTSDF; + p->volumeParams.depthTruncThreshold = rgbd::Odometry::DEFAULT_MAX_DEPTH(); + p->volumeParams.unitResolution = 16; + return p; +} + +// MatType should be Mat or UMat +template +class LargeKinfuImpl : public LargeKinfu +{ + public: + LargeKinfuImpl(const Params& _params); + virtual ~LargeKinfuImpl(); + + const Params& getParams() const CV_OVERRIDE; + + void render(OutputArray image, const Matx44f& cameraPose) const CV_OVERRIDE; + + virtual void getCloud(OutputArray points, OutputArray normals) const CV_OVERRIDE; + void getPoints(OutputArray points) const CV_OVERRIDE; + void getNormals(InputArray points, OutputArray normals) const CV_OVERRIDE; + + void reset() CV_OVERRIDE; + + const Affine3f getPose() const CV_OVERRIDE; + + bool update(InputArray depth) CV_OVERRIDE; + + bool updateT(const MatType& depth); + + private: + Params params; + + cv::Ptr icp; + //! TODO: Submap manager and Pose graph optimizer + cv::Ptr> submapMgr; + + int frameCounter; + Affine3f pose; +}; + +template +LargeKinfuImpl::LargeKinfuImpl(const Params& _params) + : params(_params) +{ + icp = makeICP(params.intr, params.icpIterations, params.icpAngleThresh, params.icpDistThresh); + + submapMgr = cv::makePtr>(params.volumeParams); + reset(); + submapMgr->createNewSubmap(true); + +} + +template +void LargeKinfuImpl::reset() +{ + frameCounter = 0; + pose = Affine3f::Identity(); + submapMgr->reset(); +} + +template +LargeKinfuImpl::~LargeKinfuImpl() +{ +} + +template +const Params& LargeKinfuImpl::getParams() const +{ + return params; +} + +template +const Affine3f LargeKinfuImpl::getPose() const +{ + return pose; +} + +template<> +bool LargeKinfuImpl::update(InputArray _depth) +{ + CV_Assert(!_depth.empty() && _depth.size() == params.frameSize); + + Mat depth; + if (_depth.isUMat()) + { + _depth.copyTo(depth); + return updateT(depth); + } + else + { + return updateT(_depth.getMat()); + } +} + +template<> +bool LargeKinfuImpl::update(InputArray _depth) +{ + CV_Assert(!_depth.empty() && _depth.size() == params.frameSize); + + UMat depth; + if (!_depth.isUMat()) + { + _depth.copyTo(depth); + return updateT(depth); + } + else + { + return updateT(_depth.getUMat()); + } +} + +template +bool LargeKinfuImpl::updateT(const MatType& _depth) +{ + CV_TRACE_FUNCTION(); + + MatType depth; + if (_depth.type() != DEPTH_TYPE) + _depth.convertTo(depth, DEPTH_TYPE); + else + depth = _depth; + + std::vector newPoints, newNormals; + makeFrameFromDepth(depth, newPoints, newNormals, params.intr, params.pyramidLevels, params.depthFactor, + params.bilateral_sigma_depth, params.bilateral_sigma_spatial, params.bilateral_kernel_size, + params.truncateThreshold); + + std::cout << "Current frameID: " << frameCounter << "\n"; + for (const auto& it : submapMgr->activeSubmaps) + { + int currTrackingId = it.first; + auto submapData = it.second; + Ptr> currTrackingSubmap = submapMgr->getSubmap(currTrackingId); + Affine3f affine; + std::cout << "Current tracking ID: " << currTrackingId << std::endl; + + if(frameCounter == 0) //! Only one current tracking map + { + currTrackingSubmap->integrate(depth, params.depthFactor, params.intr, frameCounter); + currTrackingSubmap->pyrPoints = newPoints; + currTrackingSubmap->pyrNormals = newNormals; + continue; + } + + //1. Track + bool trackingSuccess = + icp->estimateTransform(affine, currTrackingSubmap->pyrPoints, currTrackingSubmap->pyrNormals, newPoints, newNormals); + if (trackingSuccess) + currTrackingSubmap->composeCameraPose(affine); + else + { + std::cout << "Tracking failed" << std::endl; + continue; + } + + //2. Integrate + if(submapData.type == SubmapManager::Type::NEW || submapData.type == SubmapManager::Type::CURRENT) + { + float rnorm = (float)cv::norm(affine.rvec()); + float tnorm = (float)cv::norm(affine.translation()); + // We do not integrate volume if camera does not move + if ((rnorm + tnorm) / 2 >= params.tsdf_min_camera_movement) + currTrackingSubmap->integrate(depth, params.depthFactor, params.intr, frameCounter); + } + + //3. Raycast + currTrackingSubmap->raycast(currTrackingSubmap->cameraPose, params.intr, params.frameSize, currTrackingSubmap->pyrPoints[0], currTrackingSubmap->pyrNormals[0]); + + currTrackingSubmap->updatePyrPointsNormals(params.pyramidLevels); + + std::cout << "Submap: " << currTrackingId << " Total allocated blocks: " << currTrackingSubmap->getTotalAllocatedBlocks() << "\n"; + std::cout << "Submap: " << currTrackingId << " Visible blocks: " << currTrackingSubmap->getVisibleBlocks(frameCounter) << "\n"; + + } + //4. Update map + bool isMapUpdated = submapMgr->updateMap(frameCounter, newPoints, newNormals); + + if(isMapUpdated) + { + // TODO: Convert constraints to posegraph + PoseGraph poseGraph = submapMgr->MapToPoseGraph(); + std::cout << "Created posegraph\n"; + Optimizer::optimize(poseGraph); + submapMgr->PoseGraphToMap(poseGraph); + + } + std::cout << "Number of submaps: " << submapMgr->submapList.size() << "\n"; + + frameCounter++; + return true; +} + +template +void LargeKinfuImpl::render(OutputArray image, const Matx44f& _cameraPose) const +{ + CV_TRACE_FUNCTION(); + + Affine3f cameraPose(_cameraPose); + + auto currSubmap = submapMgr->getCurrentSubmap(); + const Affine3f id = Affine3f::Identity(); + if ((cameraPose.rotation() == pose.rotation() && cameraPose.translation() == pose.translation()) || + (cameraPose.rotation() == id.rotation() && cameraPose.translation() == id.translation())) + { + //! TODO: Can render be dependent on current submap + renderPointsNormals(currSubmap->pyrPoints[0], currSubmap->pyrNormals[0], image, params.lightPose); + } + else + { + MatType points, normals; + currSubmap->raycast(cameraPose, params.intr, params.frameSize, points, normals); + renderPointsNormals(points, normals, image, params.lightPose); + } +} + +template +void LargeKinfuImpl::getCloud(OutputArray p, OutputArray n) const +{ + auto currSubmap = submapMgr->getCurrentSubmap(); + currSubmap->volume.fetchPointsNormals(p, n); +} + +template +void LargeKinfuImpl::getPoints(OutputArray points) const +{ + auto currSubmap = submapMgr->getCurrentSubmap(); + currSubmap->volume.fetchPointsNormals(points, noArray()); +} + +template +void LargeKinfuImpl::getNormals(InputArray points, OutputArray normals) const +{ + auto currSubmap = submapMgr->getCurrentSubmap(); + currSubmap->volume.fetchNormals(points, normals); +} + +// importing class + +#ifdef OPENCV_ENABLE_NONFREE + +Ptr LargeKinfu::create(const Ptr& params) +{ + CV_Assert((int)params->icpIterations.size() == params->pyramidLevels); + CV_Assert(params->intr(0, 1) == 0 && params->intr(1, 0) == 0 && params->intr(2, 0) == 0 && params->intr(2, 1) == 0 && + params->intr(2, 2) == 1); +#ifdef HAVE_OPENCL + if (cv::ocl::useOpenCL()) + return makePtr>(*params); +#endif + return makePtr>(*params); +} + +#else +Ptr LargeKinfu::create(const Ptr& /* params */) +{ + CV_Error(Error::StsNotImplemented, + "This algorithm is patented and is excluded in this configuration; " + "Set OPENCV_ENABLE_NONFREE CMake option and rebuild the library"); +} +#endif +} // namespace large_kinfu +} // namespace cv diff --git a/modules/rgbd/src/pose_graph.cpp b/modules/rgbd/src/pose_graph.cpp new file mode 100644 index 00000000000..b525e455129 --- /dev/null +++ b/modules/rgbd/src/pose_graph.cpp @@ -0,0 +1,169 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "pose_graph.hpp" + +#include +#include +#include +#include + +#if defined(CERES_FOUND) +#include +#endif + +namespace cv +{ +namespace kinfu +{ +bool PoseGraph::isValid() const +{ + int numNodes = getNumNodes(); + int numEdges = getNumEdges(); + + if (numNodes <= 0 || numEdges <= 0) + return false; + + std::unordered_set nodesVisited; + std::vector nodesToVisit; + + nodesToVisit.push_back(nodes.at(0).getId()); + + bool isGraphConnected = false; + while (!nodesToVisit.empty()) + { + int currNodeId = nodesToVisit.back(); + nodesToVisit.pop_back(); + std::cout << "Visiting node: " << currNodeId << "\n"; + nodesVisited.insert(currNodeId); + // Since each node does not maintain its neighbor list + for (int i = 0; i < numEdges; i++) + { + const PoseGraphEdge& potentialEdge = edges.at(i); + int nextNodeId = -1; + + if (potentialEdge.getSourceNodeId() == currNodeId) + { + nextNodeId = potentialEdge.getTargetNodeId(); + } + else if (potentialEdge.getTargetNodeId() == currNodeId) + { + nextNodeId = potentialEdge.getSourceNodeId(); + } + if (nextNodeId != -1) + { + std::cout << "Next node: " << nextNodeId << " " << nodesVisited.count(nextNodeId) + << std::endl; + if (nodesVisited.count(nextNodeId) == 0) + { + nodesToVisit.push_back(nextNodeId); + } + } + } + } + + isGraphConnected = (int(nodesVisited.size()) == numNodes); + std::cout << "nodesVisited: " << nodesVisited.size() + << " IsGraphConnected: " << isGraphConnected << std::endl; + bool invalidEdgeNode = false; + for (int i = 0; i < numEdges; i++) + { + const PoseGraphEdge& edge = edges.at(i); + // edges have spurious source/target nodes + if ((nodesVisited.count(edge.getSourceNodeId()) != 1) || + (nodesVisited.count(edge.getTargetNodeId()) != 1)) + { + invalidEdgeNode = true; + break; + } + } + return isGraphConnected && !invalidEdgeNode; +} + +#if defined(CERES_FOUND) && defined(HAVE_EIGEN) +void Optimizer::createOptimizationProblem(PoseGraph& poseGraph, ceres::Problem& problem) +{ + int numEdges = poseGraph.getNumEdges(); + int numNodes = poseGraph.getNumNodes(); + if (numEdges == 0) + { + CV_Error(Error::StsBadArg, "PoseGraph has no edges, no optimization to be done"); + return; + } + + ceres::LossFunction* lossFunction = nullptr; + // TODO: Experiment with SE3 parameterization + ceres::LocalParameterization* quatLocalParameterization = + new ceres::EigenQuaternionParameterization; + + for (int currEdgeNum = 0; currEdgeNum < numEdges; ++currEdgeNum) + { + const PoseGraphEdge& currEdge = poseGraph.edges.at(currEdgeNum); + int sourceNodeId = currEdge.getSourceNodeId(); + int targetNodeId = currEdge.getTargetNodeId(); + Pose3d& sourcePose = poseGraph.nodes.at(sourceNodeId).se3Pose; + Pose3d& targetPose = poseGraph.nodes.at(targetNodeId).se3Pose; + + const Matx66f& informationMatrix = currEdge.information; + + ceres::CostFunction* costFunction = Pose3dErrorFunctor::create( + Pose3d(currEdge.transformation.rotation(), currEdge.transformation.translation()), + informationMatrix); + + problem.AddResidualBlock(costFunction, lossFunction, sourcePose.t.data(), + sourcePose.r.coeffs().data(), targetPose.t.data(), + targetPose.r.coeffs().data()); + problem.SetParameterization(sourcePose.r.coeffs().data(), quatLocalParameterization); + problem.SetParameterization(targetPose.r.coeffs().data(), quatLocalParameterization); + } + + for (int currNodeId = 0; currNodeId < numNodes; ++currNodeId) + { + PoseGraphNode& currNode = poseGraph.nodes.at(currNodeId); + if (currNode.isPoseFixed()) + { + problem.SetParameterBlockConstant(currNode.se3Pose.t.data()); + problem.SetParameterBlockConstant(currNode.se3Pose.r.coeffs().data()); + } + } +} +#endif + +void Optimizer::optimize(PoseGraph& poseGraph) +{ + PoseGraph poseGraphOriginal = poseGraph; + + if (!poseGraphOriginal.isValid()) + { + CV_Error(Error::StsBadArg, + "Invalid PoseGraph that is either not connected or has invalid nodes"); + return; + } + + int numNodes = poseGraph.getNumNodes(); + int numEdges = poseGraph.getNumEdges(); + std::cout << "Optimizing PoseGraph with " << numNodes << " nodes and " << numEdges << " edges" + << std::endl; + +#if defined(CERES_FOUND) && defined(HAVE_EIGEN) + ceres::Problem problem; + createOptimizationProblem(poseGraph, problem); + + ceres::Solver::Options options; + options.max_num_iterations = 100; + options.linear_solver_type = ceres::SPARSE_NORMAL_CHOLESKY; + + ceres::Solver::Summary summary; + ceres::Solve(options, &problem, &summary); + + std::cout << summary.FullReport() << '\n'; + + std::cout << "Is solution usable: " << summary.IsSolutionUsable() << std::endl; +#else + CV_Error(Error::StsNotImplemented, "Ceres and Eigen required for Pose Graph optimization"); +#endif +} + +} // namespace kinfu +} // namespace cv diff --git a/modules/rgbd/src/pose_graph.hpp b/modules/rgbd/src/pose_graph.hpp new file mode 100644 index 00000000000..bb0164723be --- /dev/null +++ b/modules/rgbd/src/pose_graph.hpp @@ -0,0 +1,321 @@ +#ifndef OPENCV_RGBD_GRAPH_NODE_H +#define OPENCV_RGBD_GRAPH_NODE_H + +#include +#include + +#include "opencv2/core/affine.hpp" +#if defined(HAVE_EIGEN) +#include +#include +#include "opencv2/core/eigen.hpp" +#endif + +#if defined(CERES_FOUND) +#include +#endif + +namespace cv +{ +namespace kinfu +{ +/*! \class GraphNode + * \brief Defines a node/variable that is optimizable in a posegraph + * + * Detailed description + */ +#if defined(HAVE_EIGEN) +struct Pose3d +{ + EIGEN_MAKE_ALIGNED_OPERATOR_NEW + + Eigen::Vector3d t; + Eigen::Quaterniond r; + + Pose3d() + { + t.setZero(); + r.setIdentity(); + }; + Pose3d(const Eigen::Matrix3d& rotation, const Eigen::Vector3d& translation) + : t(translation), r(Eigen::Quaterniond(rotation)) + { + normalizeRotation(); + } + + Pose3d(const Matx33d& rotation, const Vec3d& translation) + { + Eigen::Matrix3d R; + cv2eigen(rotation, R); + cv2eigen(translation, t); + r = Eigen::Quaterniond(R); + normalizeRotation(); + } + + explicit Pose3d(const Matx44f& pose) + { + Matx33d rotation(pose.val[0], pose.val[1], pose.val[2], pose.val[4], pose.val[5], + pose.val[6], pose.val[8], pose.val[9], pose.val[10]); + Vec3d translation(pose.val[3], pose.val[7], pose.val[11]); + Pose3d(rotation, translation); + } + + // NOTE: Eigen overloads quaternion multiplication appropriately + inline Pose3d operator*(const Pose3d& otherPose) const + { + Pose3d out(*this); + out.t += r * otherPose.t; + out.r *= otherPose.r; + out.normalizeRotation(); + return out; + } + + inline Pose3d& operator*=(const Pose3d& otherPose) + { + t += otherPose.t; + r *= otherPose.r; + normalizeRotation(); + return *this; + } + + inline Pose3d inverse() const + { + Pose3d out; + out.r = r.conjugate(); + out.t = out.r * (t * -1.0); + return out; + } + + inline void normalizeRotation() + { + if (r.w() < 0) + r.coeffs() *= -1.0; + r.normalize(); + } +}; +#endif + +struct PoseGraphNode +{ + public: + explicit PoseGraphNode(int _nodeId, const Affine3f& _pose) + : nodeId(_nodeId), isFixed(false), pose(_pose) + { +#if defined(HAVE_EIGEN) + se3Pose = Pose3d(_pose.rotation(), _pose.translation()); +#endif + } + virtual ~PoseGraphNode() = default; + + int getId() const { return nodeId; } + inline Affine3f getPose() const + { + return pose; + } + void setPose(const Affine3f& _pose) + { + pose = _pose; +#if defined(HAVE_EIGEN) + se3Pose = Pose3d(pose.rotation(), pose.translation()); +#endif + } +#if defined(HAVE_EIGEN) + void setPose(const Pose3d& _pose) + { + se3Pose = _pose; + const Eigen::Matrix3d& rotation = se3Pose.r.toRotationMatrix(); + const Eigen::Vector3d& translation = se3Pose.t; + Matx33d rot; + Vec3d trans; + eigen2cv(rotation, rot); + eigen2cv(translation, trans); + Affine3d poseMatrix(rot, trans); + pose = poseMatrix; + } +#endif + void setFixed(bool val = true) { isFixed = val; } + bool isPoseFixed() const { return isFixed; } + + public: + int nodeId; + bool isFixed; + Affine3f pose; +#if defined(HAVE_EIGEN) + Pose3d se3Pose; +#endif +}; + +/*! \class PoseGraphEdge + * \brief Defines the constraints between two PoseGraphNodes + * + * Detailed description + */ +struct PoseGraphEdge +{ + public: + PoseGraphEdge(int _sourceNodeId, int _targetNodeId, const Affine3f& _transformation, + const Matx66f& _information = Matx66f::eye()) + : sourceNodeId(_sourceNodeId), + targetNodeId(_targetNodeId), + transformation(_transformation), + information(_information) + { + } + virtual ~PoseGraphEdge() = default; + + int getSourceNodeId() const { return sourceNodeId; } + int getTargetNodeId() const { return targetNodeId; } + + bool operator==(const PoseGraphEdge& edge) + { + if ((edge.getSourceNodeId() == sourceNodeId && edge.getTargetNodeId() == targetNodeId) || + (edge.getSourceNodeId() == targetNodeId && edge.getTargetNodeId() == sourceNodeId)) + return true; + return false; + } + + public: + int sourceNodeId; + int targetNodeId; + Affine3f transformation; + Matx66f information; +}; + +//! @brief Reference: A tutorial on SE(3) transformation parameterizations and on-manifold +//! optimization Jose Luis Blanco Compactly represents the jacobian of the SE3 generator +// clang-format off +/* static const std::array generatorJacobian = { */ +/* // alpha */ +/* Matx44f(0, 0, 0, 0, */ +/* 0, 0, -1, 0, */ +/* 0, 1, 0, 0, */ +/* 0, 0, 0, 0), */ +/* // beta */ +/* Matx44f( 0, 0, 1, 0, */ +/* 0, 0, 0, 0, */ +/* -1, 0, 0, 0, */ +/* 0, 0, 0, 0), */ +/* // gamma */ +/* Matx44f(0, -1, 0, 0, */ +/* 1, 0, 0, 0, */ +/* 0, 0, 0, 0, */ +/* 0, 0, 0, 0), */ +/* // x */ +/* Matx44f(0, 0, 0, 1, */ +/* 0, 0, 0, 0, */ +/* 0, 0, 0, 0, */ +/* 0, 0, 0, 0), */ +/* // y */ +/* Matx44f(0, 0, 0, 0, */ +/* 0, 0, 0, 1, */ +/* 0, 0, 0, 0, */ +/* 0, 0, 0, 0), */ +/* // z */ +/* Matx44f(0, 0, 0, 0, */ +/* 0, 0, 0, 0, */ +/* 0, 0, 0, 1, */ +/* 0, 0, 0, 0) */ +/* }; */ +// clang-format on + +class PoseGraph +{ + public: + typedef std::vector NodeVector; + typedef std::vector EdgeVector; + + explicit PoseGraph(){}; + virtual ~PoseGraph() = default; + + //! PoseGraph can be copied/cloned + PoseGraph(const PoseGraph& _poseGraph) = default; + PoseGraph& operator=(const PoseGraph& _poseGraph) = default; + + void addNode(const PoseGraphNode& node) { nodes.push_back(node); } + void addEdge(const PoseGraphEdge& edge) { edges.push_back(edge); } + + bool nodeExists(int nodeId) const + { + return std::find_if(nodes.begin(), nodes.end(), [nodeId](const PoseGraphNode& currNode) { + return currNode.getId() == nodeId; + }) != nodes.end(); + } + + bool isValid() const; + + int getNumNodes() const { return int(nodes.size()); } + int getNumEdges() const { return int(edges.size()); } + + public: + NodeVector nodes; + EdgeVector edges; +}; + +namespace Optimizer +{ +void optimize(PoseGraph& poseGraph); + +#if defined(CERES_FOUND) +void createOptimizationProblem(PoseGraph& poseGraph, ceres::Problem& problem); + +//! Error Functor required for Ceres to obtain an auto differentiable cost function +class Pose3dErrorFunctor +{ + public: + EIGEN_MAKE_ALIGNED_OPERATOR_NEW + Pose3dErrorFunctor(const Pose3d& _poseMeasurement, const Matx66d& _sqrtInformation) + : poseMeasurement(_poseMeasurement) + { + cv2eigen(_sqrtInformation, sqrtInfo); + } + Pose3dErrorFunctor(const Pose3d& _poseMeasurement, + const Eigen::Matrix& _sqrtInformation) + : poseMeasurement(_poseMeasurement), sqrtInfo(_sqrtInformation) + { + } + + template + bool operator()(const T* const _pSourceTrans, const T* const _pSourceQuat, + const T* const _pTargetTrans, const T* const _pTargetQuat, T* _pResidual) const + { + Eigen::Map> sourceTrans(_pSourceTrans); + Eigen::Map> targetTrans(_pTargetTrans); + Eigen::Map> sourceQuat(_pSourceQuat); + Eigen::Map> targetQuat(_pTargetQuat); + Eigen::Map> residual(_pResidual); + + Eigen::Quaternion targetQuatInv = targetQuat.conjugate(); + + Eigen::Quaternion relativeQuat = targetQuatInv * sourceQuat; + Eigen::Matrix relativeTrans = targetQuatInv * (targetTrans - sourceTrans); + + //! Definition should actually be relativeQuat * poseMeasurement.r.conjugate() + Eigen::Quaternion deltaRot = + poseMeasurement.r.template cast() * relativeQuat.conjugate(); + + residual.template block<3, 1>(0, 0) = relativeTrans - poseMeasurement.t.template cast(); + residual.template block<3, 1>(3, 0) = T(2.0) * deltaRot.vec(); + + residual.applyOnTheLeft(sqrtInfo.template cast()); + + return true; + } + + static ceres::CostFunction* create(const Pose3d& _poseMeasurement, + const Matx66f& _sqrtInformation) + { + return new ceres::AutoDiffCostFunction( + new Pose3dErrorFunctor(_poseMeasurement, _sqrtInformation)); + } + + private: + const Pose3d poseMeasurement; + Eigen::Matrix sqrtInfo; +}; +#endif + +} // namespace Optimizer + +} // namespace kinfu +} // namespace cv +#endif /* ifndef OPENCV_RGBD_GRAPH_NODE_H */ diff --git a/modules/rgbd/src/sparse_block_matrix.hpp b/modules/rgbd/src/sparse_block_matrix.hpp new file mode 100644 index 00000000000..0e607af639d --- /dev/null +++ b/modules/rgbd/src/sparse_block_matrix.hpp @@ -0,0 +1,159 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include +#include + +#include "opencv2/core/base.hpp" +#include "opencv2/core/types.hpp" + +#if defined(HAVE_EIGEN) +#include +#include +#include + +#include "opencv2/core/eigen.hpp" +#endif + +namespace cv +{ +namespace kinfu +{ +/*! + * \class BlockSparseMat + * Naive implementation of Sparse Block Matrix + */ +template +struct BlockSparseMat +{ + struct Point2iHash + { + size_t operator()(const cv::Point2i& point) const noexcept + { + size_t seed = 0; + constexpr uint32_t GOLDEN_RATIO = 0x9e3779b9; + seed ^= std::hash()(point.x) + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + seed ^= std::hash()(point.y) + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + return seed; + } + }; + typedef Matx<_Tp, blockM, blockN> MatType; + typedef std::unordered_map IDtoBlockValueMap; + + BlockSparseMat(int _nBlocks) : nBlocks(_nBlocks), ijValue() {} + + MatType& refBlock(int i, int j) + { + Point2i p(i, j); + auto it = ijValue.find(p); + if (it == ijValue.end()) + { + it = ijValue.insert({ p, Matx<_Tp, blockM, blockN>::zeros() }).first; + } + return it->second; + } + + Mat diagonal() + { + // Diagonal max length is the number of columns in the sparse matrix + int diagLength = blockN * nBlocks; + cv::Mat diag = cv::Mat::zeros(diagLength, 1, CV_32F); + + for (int i = 0; i < diagLength; i++) + { + diag.at(i, 0) = refElem(i, i); + } + return diag; + } + + float& refElem(int i, int j) + { + Point2i ib(i / blockM, j / blockN), iv(i % blockM, j % blockN); + return refBlock(ib.x, ib.y)(iv.x, iv.y); + } + +#if defined(HAVE_EIGEN) + Eigen::SparseMatrix<_Tp> toEigen() const + { + std::vector> tripletList; + tripletList.reserve(ijValue.size() * blockM * blockN); + for (auto ijv : ijValue) + { + int xb = ijv.first.x, yb = ijv.first.y; + MatType vblock = ijv.second; + for (int i = 0; i < blockM; i++) + { + for (int j = 0; j < blockN; j++) + { + float val = vblock(i, j); + if (abs(val) >= NON_ZERO_VAL_THRESHOLD) + { + tripletList.push_back(Eigen::Triplet(blockM * xb + i, blockN * yb + j, val)); + } + } + } + } + Eigen::SparseMatrix<_Tp> EigenMat(blockM * nBlocks, blockN * nBlocks); + EigenMat.setFromTriplets(tripletList.begin(), tripletList.end()); + EigenMat.makeCompressed(); + + return EigenMat; + } +#endif + size_t nonZeroBlocks() const { return ijValue.size(); } + + static constexpr float NON_ZERO_VAL_THRESHOLD = 0.0001f; + int nBlocks; + IDtoBlockValueMap ijValue; +}; + +//! Function to solve a sparse linear system of equations HX = B +//! Requires Eigen +static bool sparseSolve(const BlockSparseMat& H, const Mat& B, Mat& X, Mat& predB) +{ + bool result = false; +#if defined(HAVE_EIGEN) + Eigen::SparseMatrix bigA = H.toEigen(); + Eigen::VectorXf bigB; + cv2eigen(B, bigB); + + Eigen::SparseMatrix bigAtranspose = bigA.transpose(); + if(!bigA.isApprox(bigAtranspose)) + { + CV_Error(Error::StsBadArg, "H matrix is not symmetrical"); + return result; + } + + Eigen::SimplicialLDLT> solver; + + solver.compute(bigA); + if (solver.info() != Eigen::Success) + { + std::cout << "failed to eigen-decompose" << std::endl; + result = false; + } + else + { + Eigen::VectorXf solutionX = solver.solve(bigB); + Eigen::VectorXf predBEigen = bigA * solutionX; + if (solver.info() != Eigen::Success) + { + std::cout << "failed to eigen-solve" << std::endl; + result = false; + } + else + { + eigen2cv(solutionX, X); + eigen2cv(predBEigen, predB); + result = true; + } + } +#else + std::cout << "no eigen library" << std::endl; + CV_Error(Error::StsNotImplemented, "Eigen library required for matrix solve, dense solver is not implemented"); +#endif + return result; +} +} // namespace kinfu +} // namespace cv diff --git a/modules/rgbd/src/submap.hpp b/modules/rgbd/src/submap.hpp new file mode 100644 index 00000000000..65a2bed6229 --- /dev/null +++ b/modules/rgbd/src/submap.hpp @@ -0,0 +1,544 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef __OPENCV_RGBD_SUBMAP_HPP__ +#define __OPENCV_RGBD_SUBMAP_HPP__ + +#include + +#include +#include +#include + +#include "hash_tsdf.hpp" +#include "opencv2/core/mat.inl.hpp" +#include "pose_graph.hpp" + +namespace cv +{ +namespace kinfu +{ +template +class Submap +{ + public: + struct PoseConstraint + { + Affine3f estimatedPose; + int weight; + + PoseConstraint() : weight(0){}; + + void accumulatePose(const Affine3f& _pose, int _weight = 1) + { + Matx44f accPose = estimatedPose.matrix * weight + _pose.matrix * _weight; + weight += _weight; + accPose /= float(weight); + estimatedPose = Affine3f(accPose); + } + }; + typedef std::map Constraints; + + Submap(int _id, const VolumeParams& volumeParams, const cv::Affine3f& _pose = cv::Affine3f::Identity(), + int _startFrameId = 0) + : id(_id), pose(_pose), cameraPose(Affine3f::Identity()), startFrameId(_startFrameId), volume(volumeParams) + { + std::cout << "Created volume\n"; + } + virtual ~Submap() = default; + + virtual void integrate(InputArray _depth, float depthFactor, const cv::kinfu::Intr& intrinsics, const int currframeId); + virtual void raycast(const cv::Affine3f& cameraPose, const cv::kinfu::Intr& intrinsics, cv::Size frameSize, + OutputArray points, OutputArray normals); + virtual void updatePyrPointsNormals(const int pyramidLevels); + + virtual int getTotalAllocatedBlocks() const { return int(volume.getTotalVolumeUnits()); }; + virtual int getVisibleBlocks(int currFrameId) const + { + return volume.getVisibleBlocks(currFrameId, FRAME_VISIBILITY_THRESHOLD); + } + + float calcVisibilityRatio(int currFrameId) const + { + int allocate_blocks = getTotalAllocatedBlocks(); + int visible_blocks = getVisibleBlocks(currFrameId); + return float(visible_blocks) / float(allocate_blocks); + } + + //! TODO: Possibly useless + virtual void setStartFrameId(int _startFrameId) { startFrameId = _startFrameId; }; + virtual void setStopFrameId(int _stopFrameId) { stopFrameId = _stopFrameId; }; + + void composeCameraPose(const cv::Affine3f& _relativePose) { cameraPose = cameraPose * _relativePose; } + PoseConstraint& getConstraint(const int _id) + { + //! Creates constraints if doesn't exist yet + return constraints[_id]; + } + + public: + const int id; + cv::Affine3f pose; + cv::Affine3f cameraPose; + Constraints constraints; + + int startFrameId; + int stopFrameId; + //! TODO: Should we support submaps for regular volumes? + static constexpr int FRAME_VISIBILITY_THRESHOLD = 5; + + //! TODO: Add support for GPU arrays (UMat) + std::vector pyrPoints; + std::vector pyrNormals; + HashTSDFVolumeCPU volume; +}; + +template + +void Submap::integrate(InputArray _depth, float depthFactor, const cv::kinfu::Intr& intrinsics, + const int currFrameId) +{ + CV_Assert(currFrameId >= startFrameId); + volume.integrate(_depth, depthFactor, cameraPose.matrix, intrinsics, currFrameId); +} + +template +void Submap::raycast(const cv::Affine3f& _cameraPose, const cv::kinfu::Intr& intrinsics, cv::Size frameSize, + OutputArray points, OutputArray normals) +{ + volume.raycast(_cameraPose.matrix, intrinsics, frameSize, points, normals); +} + +template +void Submap::updatePyrPointsNormals(const int pyramidLevels) +{ + MatType& points = pyrPoints[0]; + MatType& normals = pyrNormals[0]; + + buildPyramidPointsNormals(points, normals, pyrPoints, pyrNormals, pyramidLevels); +} + +/** + * @brief: Manages all the created submaps for a particular scene + */ +template +class SubmapManager +{ + public: + enum class Type + { + NEW = 0, + CURRENT = 1, + RELOCALISATION = 2, + LOOP_CLOSURE = 3, + LOST = 4 + }; + + struct ActiveSubmapData + { + Type type; + std::vector constraints; + int trackingAttempts; + }; + typedef Submap SubmapT; + typedef std::map> IdToSubmapPtr; + typedef std::unordered_map IdToActiveSubmaps; + + SubmapManager(const VolumeParams& _volumeParams) : volumeParams(_volumeParams) {} + virtual ~SubmapManager() = default; + + void reset() { submapList.clear(); }; + + bool shouldCreateSubmap(int frameId); + bool shouldChangeCurrSubmap(int _frameId, int toSubmapId); + + //! Adds a new submap/volume into the current list of managed/Active submaps + int createNewSubmap(bool isCurrentActiveMap, const int currFrameId = 0, const Affine3f& pose = cv::Affine3f::Identity()); + + void removeSubmap(int _id); + size_t numOfSubmaps(void) const { return submapList.size(); }; + size_t numOfActiveSubmaps(void) const { return activeSubmaps.size(); }; + + Ptr getSubmap(int _id) const; + Ptr getCurrentSubmap(void) const; + + int estimateConstraint(int fromSubmapId, int toSubmapId, int& inliers, Affine3f& inlierPose); + bool updateMap(int _frameId, std::vector _framePoints, std::vector _frameNormals); + + PoseGraph MapToPoseGraph(); + void PoseGraphToMap(const PoseGraph& updatedPoseGraph); + + VolumeParams volumeParams; + + std::vector> submapList; + IdToActiveSubmaps activeSubmaps; + + PoseGraph poseGraph; +}; + +template +int SubmapManager::createNewSubmap(bool isCurrentMap, int currFrameId, const Affine3f& pose) +{ + int newId = int(submapList.size()); + + Ptr newSubmap = cv::makePtr(newId, volumeParams, pose, currFrameId); + submapList.push_back(newSubmap); + + ActiveSubmapData newSubmapData; + newSubmapData.trackingAttempts = 0; + newSubmapData.type = isCurrentMap ? Type::CURRENT : Type::NEW; + activeSubmaps[newId] = newSubmapData; + + std::cout << "Created new submap\n"; + + return newId; +} + +template +Ptr> SubmapManager::getSubmap(int _id) const +{ + CV_Assert(submapList.size() > 0); + CV_Assert(_id >= 0 && _id < int(submapList.size())); + return submapList.at(_id); +} + +template +Ptr> SubmapManager::getCurrentSubmap(void) const +{ + for (const auto& it : activeSubmaps) + { + if (it.second.type == Type::CURRENT) + return getSubmap(it.first); + } + return nullptr; +} + +template +bool SubmapManager::shouldCreateSubmap(int currFrameId) +{ + int currSubmapId = -1; + for (const auto& it : activeSubmaps) + { + auto submapData = it.second; + // No more than 1 new submap at a time! + if (submapData.type == Type::NEW) + { + return false; + } + if (submapData.type == Type::CURRENT) + { + currSubmapId = it.first; + } + } + //! TODO: This shouldn't be happening? since there should always be one active current submap + if (currSubmapId < 0) + { + return false; + } + + Ptr currSubmap = getSubmap(currSubmapId); + float ratio = currSubmap->calcVisibilityRatio(currFrameId); + + std::cout << "Ratio: " << ratio << "\n"; + + if (ratio < 0.2f) + return true; + return false; +} + +template +int SubmapManager::estimateConstraint(int fromSubmapId, int toSubmapId, int& inliers, Affine3f& inlierPose) +{ + static constexpr int MAX_ITER = 10; + static constexpr float CONVERGE_WEIGHT_THRESHOLD = 0.01f; + static constexpr float INLIER_WEIGHT_THRESH = 0.8f; + static constexpr int MIN_INLIERS = 10; + static constexpr int MAX_TRACKING_ATTEMPTS = 25; + + //! thresh = HUBER_THRESH + auto huberWeight = [](float residual, float thresh = 0.1f) -> float { + float rAbs = abs(residual); + if (rAbs < thresh) + return 1.0; + float numerator = sqrt(2 * thresh * rAbs - thresh * thresh); + return numerator / rAbs; + }; + + Ptr fromSubmap = getSubmap(fromSubmapId); + Ptr toSubmap = getSubmap(toSubmapId); + ActiveSubmapData& fromSubmapData = activeSubmaps.at(fromSubmapId); + + Affine3f TcameraToFromSubmap = fromSubmap->cameraPose; + Affine3f TcameraToToSubmap = toSubmap->cameraPose; + + // FromSubmap -> ToSubmap transform + Affine3f candidateConstraint = TcameraToToSubmap * TcameraToFromSubmap.inv(); + fromSubmapData.trackingAttempts++; + fromSubmapData.constraints.push_back(candidateConstraint); + + std::vector weights(fromSubmapData.constraints.size() + 1, 1.0f); + + Affine3f prevConstraint = fromSubmap->getConstraint(toSubmap->id).estimatedPose; + int prevWeight = fromSubmap->getConstraint(toSubmap->id).weight; + + // Iterative reweighted least squares with huber threshold to find the inliers in the past observations + Vec6f meanConstraint; + float sumWeight = 0.0f; + for (int i = 0; i < MAX_ITER; i++) + { + Vec6f constraintVec; + for (int j = 0; j < int(weights.size() - 1); j++) + { + Affine3f currObservation = fromSubmapData.constraints[j]; + cv::vconcat(currObservation.rvec(), currObservation.translation(), constraintVec); + meanConstraint += weights[j] * constraintVec; + sumWeight += weights[j]; + } + // Heavier weight given to the estimatedPose + cv::vconcat(prevConstraint.rvec(), prevConstraint.translation(), constraintVec); + meanConstraint += weights.back() * prevWeight * constraintVec; + sumWeight += prevWeight; + meanConstraint /= float(sumWeight); + + float residual = 0.0f; + float diff = 0.0f; + for (int j = 0; j < int(weights.size()); j++) + { + int w; + if (j == int(weights.size() - 1)) + { + cv::vconcat(prevConstraint.rvec(), prevConstraint.translation(), constraintVec); + w = prevWeight; + } + else + { + Affine3f currObservation = fromSubmapData.constraints[j]; + cv::vconcat(currObservation.rvec(), currObservation.translation(), constraintVec); + w = 1; + } + + cv::Vec6f residualVec = (constraintVec - meanConstraint); + residual = float(norm(residualVec)); + float newWeight = huberWeight(residual); + diff += w * abs(newWeight - weights[j]); + weights[j] = newWeight; + } + + if (diff / (prevWeight + weights.size() - 1) < CONVERGE_WEIGHT_THRESHOLD) + break; + } + + int localInliers = 0; + Matx44f inlierConstraint; + for (int i = 0; i < int(weights.size()); i++) + { + if (weights[i] > INLIER_WEIGHT_THRESH) + { + localInliers++; + if (i == int(weights.size() - 1)) + inlierConstraint += prevConstraint.matrix; + else + inlierConstraint += fromSubmapData.constraints[i].matrix; + } + } + inlierConstraint /= float(max(localInliers, 1)); + inlierPose = Affine3f(inlierConstraint); + inliers = localInliers; + + /* std::cout << inlierPose.matrix << "\n"; */ + /* std::cout << " inliers: " << inliers << "\n"; */ + + if (inliers >= MIN_INLIERS) + { + return 1; + } + if(fromSubmapData.trackingAttempts - inliers > (MAX_TRACKING_ATTEMPTS - MIN_INLIERS)) + { + return -1; + } + + return 0; +} + +template +bool SubmapManager::shouldChangeCurrSubmap(int _frameId, int toSubmapId) +{ + auto toSubmap = getSubmap(toSubmapId); + auto toSubmapData = activeSubmaps.at(toSubmapId); + auto currActiveSubmap = getCurrentSubmap(); + + int blocksInNewMap = toSubmap->getTotalAllocatedBlocks(); + float newRatio = toSubmap->calcVisibilityRatio(_frameId); + + float currRatio = currActiveSubmap->calcVisibilityRatio(_frameId); + + //! TODO: Check for a specific threshold? + if (blocksInNewMap <= 0) + return false; + if ((newRatio > currRatio) && (toSubmapData.type == Type::NEW)) + return true; + + return false; +} + +template +bool SubmapManager::updateMap(int _frameId, std::vector _framePoints, std::vector _frameNormals) +{ + bool mapUpdated = false; + int changedCurrentMapId = -1; + + const int currSubmapId = getCurrentSubmap()->id; + + for (auto& it : activeSubmaps) + { + int submapId = it.first; + auto& submapData = it.second; + if (submapData.type == Type::NEW || submapData.type == Type::LOOP_CLOSURE) + { + // Check with previous estimate + int inliers; + Affine3f inlierPose; + int constraintUpdate = estimateConstraint(submapId, currSubmapId, inliers, inlierPose); + std::cout << "SubmapId: " << submapId << " Tracking attempts: " << submapData.trackingAttempts << "\n"; + if (constraintUpdate == 1) + { + typename SubmapT::PoseConstraint& submapConstraint = getSubmap(submapId)->getConstraint(currSubmapId); + submapConstraint.accumulatePose(inlierPose, inliers); + std::cout << "Submap constraint estimated pose: \n" << submapConstraint.estimatedPose.matrix << "\n"; + submapData.constraints.clear(); + submapData.trackingAttempts = 0; + + if (shouldChangeCurrSubmap(_frameId, submapId)) + { + std::cout << "Should change current map to the new map\n"; + changedCurrentMapId = submapId; + } + mapUpdated = true; + } + else if(constraintUpdate == -1) + { + submapData.type = Type::LOST; + } + } + } + + std::vector createNewConstraintsList; + for (auto& it : activeSubmaps) + { + int submapId = it.first; + auto& submapData = it.second; + + if (submapId == changedCurrentMapId) + { + submapData.type = Type::CURRENT; + } + if ((submapData.type == Type::CURRENT) && (changedCurrentMapId >= 0) && (submapId != changedCurrentMapId)) + { + submapData.type = Type::LOST; + createNewConstraintsList.push_back(submapId); + } + if ((submapData.type == Type::NEW || submapData.type == Type::LOOP_CLOSURE) && (changedCurrentMapId >= 0)) + { + //! TODO: Add a new type called NEW_LOST? + submapData.type = Type::LOST; + createNewConstraintsList.push_back(submapId); + } + } + + for (typename IdToActiveSubmaps::iterator it = activeSubmaps.begin(); it != activeSubmaps.end();) + { + auto& submapData = it->second; + if (submapData.type == Type::LOST) + it = activeSubmaps.erase(it); + else + it++; + } + + for (std::vector::const_iterator it = createNewConstraintsList.begin(); it != createNewConstraintsList.end(); ++it) + { + int dataId = *it; + ActiveSubmapData newSubmapData; + newSubmapData.trackingAttempts = 0; + newSubmapData.type = Type::LOOP_CLOSURE; + activeSubmaps[dataId] = newSubmapData; + } + + if (shouldCreateSubmap(_frameId)) + { + Ptr currActiveSubmap = getCurrentSubmap(); + Affine3f newSubmapPose = currActiveSubmap->pose * currActiveSubmap->cameraPose; + int submapId = createNewSubmap(false, _frameId, newSubmapPose); + auto newSubmap = getSubmap(submapId); + newSubmap->pyrPoints = _framePoints; + newSubmap->pyrNormals = _frameNormals; + } + + // Debugging only + if(_frameId%100 == 0) + { + for(size_t i = 0; i < submapList.size(); i++) + { + Ptr currSubmap = submapList.at(i); + typename SubmapT::Constraints::const_iterator itBegin = currSubmap->constraints.begin(); + std::cout << "Constraint list for SubmapID: " << currSubmap->id << "\n"; + for(typename SubmapT::Constraints::const_iterator it = itBegin; it != currSubmap->constraints.end(); ++it) + { + const typename SubmapT::PoseConstraint& constraint = it->second; + std::cout << "[" << it->first << "] weight: " << constraint.weight << "\n " << constraint.estimatedPose.matrix << " \n"; + } + } + } + + return mapUpdated; +} + +template +PoseGraph SubmapManager::MapToPoseGraph() +{ + PoseGraph localPoseGraph; + + + for(const auto& currSubmap : submapList) + { + const typename SubmapT::Constraints& constraintList = currSubmap->constraints; + for(const auto& currConstraintPair : constraintList) + { + // TODO: Handle case with duplicate constraints A -> B and B -> A + /* Matx66f informationMatrix = Matx66f::eye() * (currConstraintPair.second.weight/10); */ + Matx66f informationMatrix = Matx66f::eye(); + PoseGraphEdge currEdge(currSubmap->id, currConstraintPair.first, currConstraintPair.second.estimatedPose, informationMatrix); + localPoseGraph.addEdge(currEdge); + } + } + + for(const auto& currSubmap : submapList) + { + PoseGraphNode currNode(currSubmap->id, currSubmap->pose); + if(currSubmap->id == 0) + { + currNode.setFixed(); + } + localPoseGraph.addNode(currNode); + } + + + + return localPoseGraph; +} + +template +void SubmapManager::PoseGraphToMap(const PoseGraph &updatedPoseGraph) +{ + for(const auto& currSubmap : submapList) + { + const PoseGraphNode& currNode = updatedPoseGraph.nodes.at(currSubmap->id); + if(!currNode.isPoseFixed()) + currSubmap->pose = currNode.getPose(); + std::cout << "Current node: " << currSubmap->id << " Updated Pose: \n" << currSubmap->pose.matrix << std::endl; + } +} + +} // namespace kinfu +} // namespace cv +#endif /* ifndef __OPENCV_RGBD_SUBMAP_HPP__ */ diff --git a/modules/rgbd/src/tsdf.cpp b/modules/rgbd/src/tsdf.cpp index 947428065d1..1e8704170f4 100644 --- a/modules/rgbd/src/tsdf.cpp +++ b/modules/rgbd/src/tsdf.cpp @@ -5,32 +5,14 @@ // This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory #include "precomp.hpp" -#include "tsdf.hpp" +//#include "tsdf.hpp" +#include "tsdf_functions.hpp" #include "opencl_kernels_rgbd.hpp" namespace cv { namespace kinfu { -static inline v_float32x4 tsdfToFloat_INTR(const v_int32x4& num) -{ - v_float32x4 num128 = v_setall_f32(-1.f / 128.f); - return v_cvt_f32(num) * num128; -} - -static inline TsdfType floatToTsdf(float num) -{ - //CV_Assert(-1 < num <= 1); - int8_t res = int8_t(num * (-128.f)); - res = res ? res : (num < 0 ? 1 : -1); - return res; -} - -static inline float tsdfToFloat(TsdfType num) -{ - return float(num) * (-1.f / 128.f); -} - TSDFVolume::TSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, Point3i _resolution, bool zFirstMemOrder) : Volume(_voxelSize, _pose, _raycastStepFactor), @@ -81,6 +63,21 @@ TSDFVolumeCPU::TSDFVolumeCPU(float _voxelSize, cv::Matx44f _pose, float _raycast : TSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _resolution, zFirstMemOrder) { + int xdim, ydim, zdim; + if (zFirstMemOrder) + { + xdim = volResolution.z * volResolution.y; + ydim = volResolution.z; + zdim = 1; + } + else + { + xdim = 1; + ydim = volResolution.x; + zdim = volResolution.x * volResolution.y; + } + volStrides = Vec4i(xdim, ydim, zdim); + volume = Mat(1, volResolution.x * volResolution.y * volResolution.z, rawType()); reset(); @@ -98,7 +95,7 @@ void TSDFVolumeCPU::reset() }); } -TsdfVoxel TSDFVolumeCPU::at(const cv::Vec3i& volumeIdx) const +TsdfVoxel TSDFVolumeCPU::at(const Vec3i& volumeIdx) const { //! Out of bounds if ((volumeIdx[0] >= volResolution.x || volumeIdx[0] < 0) || @@ -117,403 +114,32 @@ TsdfVoxel TSDFVolumeCPU::at(const cv::Vec3i& volumeIdx) const return volData[coordBase]; } -// SIMD version of that code is manually inlined -#if !USE_INTRINSICS -static const bool fixMissingData = false; - -static inline depthType bilinearDepth(const Depth& m, cv::Point2f pt) -{ - const depthType defaultValue = qnan; - if(pt.x < 0 || pt.x >= m.cols-1 || - pt.y < 0 || pt.y >= m.rows-1) - return defaultValue; - - int xi = cvFloor(pt.x), yi = cvFloor(pt.y); - - const depthType* row0 = m[yi+0]; - const depthType* row1 = m[yi+1]; - - depthType v00 = row0[xi+0]; - depthType v01 = row0[xi+1]; - depthType v10 = row1[xi+0]; - depthType v11 = row1[xi+1]; - - // assume correct depth is positive - bool b00 = v00 > 0; - bool b01 = v01 > 0; - bool b10 = v10 > 0; - bool b11 = v11 > 0; - - if(!fixMissingData) - { - if(!(b00 && b01 && b10 && b11)) - return defaultValue; - else - { - float tx = pt.x - xi, ty = pt.y - yi; - depthType v0 = v00 + tx*(v01 - v00); - depthType v1 = v10 + tx*(v11 - v10); - return v0 + ty*(v1 - v0); - } - } - else - { - int nz = b00 + b01 + b10 + b11; - if(nz == 0) - { - return defaultValue; - } - if(nz == 1) - { - if(b00) return v00; - if(b01) return v01; - if(b10) return v10; - if(b11) return v11; - } - if(nz == 2) - { - if(b00 && b10) v01 = v00, v11 = v10; - if(b01 && b11) v00 = v01, v10 = v11; - if(b00 && b01) v10 = v00, v11 = v01; - if(b10 && b11) v00 = v10, v01 = v11; - if(b00 && b11) v01 = v10 = (v00 + v11)*0.5f; - if(b01 && b10) v00 = v11 = (v01 + v10)*0.5f; - } - if(nz == 3) - { - if(!b00) v00 = v10 + v01 - v11; - if(!b01) v01 = v00 + v11 - v10; - if(!b10) v10 = v00 + v11 - v01; - if(!b11) v11 = v01 + v10 - v00; - } - - float tx = pt.x - xi, ty = pt.y - yi; - depthType v0 = v00 + tx*(v01 - v00); - depthType v1 = v10 + tx*(v11 - v10); - return v0 + ty*(v1 - v0); - } -} -#endif - - - -struct IntegrateInvoker : ParallelLoopBody -{ - IntegrateInvoker(TSDFVolumeCPU& _volume, const Depth& _depth, const Intr& intrinsics, - const cv::Matx44f& cameraPose, float depthFactor, Mat _pixNorms) : - ParallelLoopBody(), - volume(_volume), - depth(_depth), - intr(intrinsics), - proj(intrinsics.makeProjector()), - vol2cam(Affine3f(cameraPose.inv()) * _volume.pose), - truncDistInv(1.f/_volume.truncDist), - dfac(1.f/depthFactor), - pixNorms(_pixNorms) - { - volDataStart = volume.volume.ptr(); - } - -#if USE_INTRINSICS - virtual void operator() (const Range& range) const override - { - // zStep == vol2cam*(Point3f(x, y, 1)*voxelSize) - basePt; - Point3f zStepPt = Point3f(vol2cam.matrix(0, 2), - vol2cam.matrix(1, 2), - vol2cam.matrix(2, 2))*volume.voxelSize; - - v_float32x4 zStep(zStepPt.x, zStepPt.y, zStepPt.z, 0); - v_float32x4 vfxy(proj.fx, proj.fy, 0.f, 0.f), vcxy(proj.cx, proj.cy, 0.f, 0.f); - const v_float32x4 upLimits = v_cvt_f32(v_int32x4(depth.cols-1, depth.rows-1, 0, 0)); - - for(int x = range.start; x < range.end; x++) - { - TsdfVoxel* volDataX = volDataStart + x*volume.volDims[0]; - for(int y = 0; y < volume.volResolution.y; y++) - { - TsdfVoxel* volDataY = volDataX + y*volume.volDims[1]; - // optimization of camSpace transformation (vector addition instead of matmul at each z) - Point3f basePt = vol2cam*(Point3f((float)x, (float)y, 0)*volume.voxelSize); - v_float32x4 camSpacePt(basePt.x, basePt.y, basePt.z, 0); - - int startZ, endZ; - if(abs(zStepPt.z) > 1e-5) - { - int baseZ = (int)(-basePt.z / zStepPt.z); - if(zStepPt.z > 0) - { - startZ = baseZ; - endZ = volume.volResolution.z; - } - else - { - startZ = 0; - endZ = baseZ; - } - } - else - { - if (basePt.z > 0) - { - startZ = 0; - endZ = volume.volResolution.z; - } - else - { - // z loop shouldn't be performed - startZ = endZ = 0; - } - } - startZ = max(0, startZ); - endZ = min(volume.volResolution.z, endZ); - for(int z = startZ; z < endZ; z++) - { - // optimization of the following: - //Point3f volPt = Point3f(x, y, z)*voxelSize; - //Point3f camSpacePt = vol2cam * volPt; - camSpacePt += zStep; - - float zCamSpace = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))).get0(); - if(zCamSpace <= 0.f) - continue; - - v_float32x4 camPixVec = camSpacePt/v_setall_f32(zCamSpace); - v_float32x4 projected = v_muladd(camPixVec, vfxy, vcxy); - // leave only first 2 lanes - projected = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) & - v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)); - - depthType v; - // bilinearly interpolate depth at projected - { - const v_float32x4& pt = projected; - // check coords >= 0 and < imgSize - v_uint32x4 limits = v_reinterpret_as_u32(pt < v_setzero_f32()) | - v_reinterpret_as_u32(pt >= upLimits); - limits = limits | v_rotate_right<1>(limits); - if(limits.get0()) - continue; - - // xi, yi = floor(pt) - v_int32x4 ip = v_floor(pt); - v_int32x4 ipshift = ip; - int xi = ipshift.get0(); - ipshift = v_rotate_right<1>(ipshift); - int yi = ipshift.get0(); - - const depthType* row0 = depth[yi+0]; - const depthType* row1 = depth[yi+1]; - - // v001 = [v(xi + 0, yi + 0), v(xi + 1, yi + 0)] - v_float32x4 v001 = v_load_low(row0 + xi); - // v101 = [v(xi + 0, yi + 1), v(xi + 1, yi + 1)] - v_float32x4 v101 = v_load_low(row1 + xi); - - v_float32x4 vall = v_combine_low(v001, v101); - - // assume correct depth is positive - // don't fix missing data - if(v_check_all(vall > v_setzero_f32())) - { - v_float32x4 t = pt - v_cvt_f32(ip); - float tx = t.get0(); - t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - v_float32x4 ty = v_setall_f32(t.get0()); - // vx is y-interpolated between rows 0 and 1 - v_float32x4 vx = v001 + ty*(v101 - v001); - float v0 = vx.get0(); - vx = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vx))); - float v1 = vx.get0(); - v = v0 + tx*(v1 - v0); - } - else - continue; - } - - // norm(camPixVec) produces double which is too slow - int _u = (int) projected.get0(); - int _v = (int) v_rotate_right<1>(projected).get0(); - if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows)) - continue; - float pixNorm = pixNorms.at(_v, _u); - // float pixNorm = sqrt(v_reduce_sum(camPixVec*camPixVec)); - // difference between distances of point and of surface to camera - float sdf = pixNorm*(v*dfac - zCamSpace); - // possible alternative is: - // kftype sdf = norm(camSpacePt)*(v*dfac/camSpacePt.z - 1); - if(sdf >= -volume.truncDist) - { - TsdfType tsdf = floatToTsdf(fmin(1.f, sdf * truncDistInv)); - - TsdfVoxel& voxel = volDataY[z*volume.volDims[2]]; - WeightType& weight = voxel.weight; - TsdfType& value = voxel.tsdf; - - // update TSDF - value = floatToTsdf((tsdfToFloat(value)*weight+ tsdfToFloat(tsdf)) / (weight + 1)); - weight = (weight + 1) < volume.maxWeight ? (weight + 1) : volume.maxWeight; - } - } - } - } - } -#else - virtual void operator() (const Range& range) const override - { - for(int x = range.start; x < range.end; x++) - { - TsdfVoxel* volDataX = volDataStart + x*volume.volDims[0]; - for(int y = 0; y < volume.volResolution.y; y++) - { - TsdfVoxel* volDataY = volDataX+y*volume.volDims[1]; - // optimization of camSpace transformation (vector addition instead of matmul at each z) - Point3f basePt = vol2cam*(Point3f(float(x), float(y), 0.0f)*volume.voxelSize); - Point3f camSpacePt = basePt; - // zStep == vol2cam*(Point3f(x, y, 1)*voxelSize) - basePt; - // zStep == vol2cam*[Point3f(x, y, 1) - Point3f(x, y, 0)]*voxelSize - Point3f zStep = Point3f(vol2cam.matrix(0, 2), - vol2cam.matrix(1, 2), - vol2cam.matrix(2, 2))*volume.voxelSize; - int startZ, endZ; - if(abs(zStep.z) > 1e-5) - { - int baseZ = int(-basePt.z / zStep.z); - if(zStep.z > 0) - { - startZ = baseZ; - endZ = volume.volResolution.z; - } - else - { - startZ = 0; - endZ = baseZ; - } - } - else - { - if(basePt.z > 0) - { - startZ = 0; - endZ = volume.volResolution.z; - } - else - { - // z loop shouldn't be performed - startZ = endZ = 0; - } - } - startZ = max(0, startZ); - endZ = min(volume.volResolution.z, endZ); - - for (int z = startZ; z < endZ; z++) - { - // optimization of the following: - //Point3f volPt = Point3f(x, y, z)*volume.voxelSize; - //Point3f camSpacePt = vol2cam * volPt; - - camSpacePt += zStep; - if(camSpacePt.z <= 0) - continue; - - Point3f camPixVec; - Point2f projected = proj(camSpacePt, camPixVec); - - depthType v = bilinearDepth(depth, projected); - if (v == 0) { - continue; - } - - int _u = projected.x; - int _v = projected.y; - if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows)) - continue; - float pixNorm = pixNorms.at(_v, _u); - - // difference between distances of point and of surface to camera - float sdf = pixNorm*(v*dfac - camSpacePt.z); - // possible alternative is: - // kftype sdf = norm(camSpacePt)*(v*dfac/camSpacePt.z - 1); - if(sdf >= -volume.truncDist) - { - TsdfType tsdf = floatToTsdf(fmin(1.f, sdf * truncDistInv)); - - TsdfVoxel& voxel = volDataY[z*volume.volDims[2]]; - WeightType& weight = voxel.weight; - TsdfType& value = voxel.tsdf; - - // update TSDF - value = floatToTsdf((tsdfToFloat(value)*weight+ tsdfToFloat(tsdf)) / (weight + 1)); - weight = min(int(weight + 1), int(volume.maxWeight)); - } - } - } - } - } -#endif - - TSDFVolumeCPU& volume; - const Depth& depth; - const Intr& intr; - const Intr::Projector proj; - const cv::Affine3f vol2cam; - const float truncDistInv; - const float dfac; - TsdfVoxel* volDataStart; - Mat pixNorms; -}; - -static cv::Mat preCalculationPixNorm(Depth depth, const Intr& intrinsics) -{ - int height = depth.rows; - int widht = depth.cols; - Point2f fl(intrinsics.fx, intrinsics.fy); - Point2f pp(intrinsics.cx, intrinsics.cy); - Mat pixNorm (height, widht, CV_32F); - std::vector x(widht); - std::vector y(height); - for (int i = 0; i < widht; i++) - x[i] = (i - pp.x) / fl.x; - for (int i = 0; i < height; i++) - y[i] = (i - pp.y) / fl.y; - - for (int i = 0; i < height; i++) - { - for (int j = 0; j < widht; j++) - { - pixNorm.at(i, j) = sqrtf(x[j] * x[j] + y[i] * y[i] + 1.0f); - } - } - return pixNorm; -} - // use depth instead of distance (optimization) -void TSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, - const Intr& intrinsics) +void TSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, + const Intr& intrinsics, const int frameId) { CV_TRACE_FUNCTION(); - + CV_UNUSED(frameId); CV_Assert(_depth.type() == DEPTH_TYPE); CV_Assert(!_depth.empty()); Depth depth = _depth.getMat(); - if (!(frameParams[0] == depth.rows && frameParams[1] == depth.cols && - frameParams[2] == intrinsics.fx && frameParams[3] == intrinsics.fy && - frameParams[4] == intrinsics.cx && frameParams[5] == intrinsics.cy)) - { - frameParams[0] = (float)depth.rows; frameParams[1] = (float)depth.cols; - frameParams[2] = intrinsics.fx; frameParams[3] = intrinsics.fy; - frameParams[4] = intrinsics.cx; frameParams[5] = intrinsics.cy; + Vec6f newParams((float)depth.rows, (float)depth.cols, + intrinsics.fx, intrinsics.fy, + intrinsics.cx, intrinsics.cy); + if (!(frameParams == newParams)) + { + frameParams = newParams; pixNorms = preCalculationPixNorm(depth, intrinsics); } - IntegrateInvoker ii(*this, depth, intrinsics, cameraPose, depthFactor, pixNorms); - Range range(0, volResolution.x); - parallel_for_(range, ii); + integrateVolumeUnit(truncDist, voxelSize, maxWeight, (this->pose).matrix, volResolution, volStrides, depth, + depthFactor, cameraPose, intrinsics, pixNorms, volume); } #if USE_INTRINSICS // all coordinate checks should be done in inclosing cycle -inline float TSDFVolumeCPU::interpolateVoxel(Point3f _p) const +inline float TSDFVolumeCPU::interpolateVoxel(const Point3f& _p) const { v_float32x4 p(_p.x, _p.y, _p.z, 0); return interpolateVoxel(p); @@ -560,7 +186,7 @@ inline float TSDFVolumeCPU::interpolateVoxel(const v_float32x4& p) const return v0 + tx*(v1 - v0); } #else -inline float TSDFVolumeCPU::interpolateVoxel(Point3f p) const +inline float TSDFVolumeCPU::interpolateVoxel(const Point3f& p) const { int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; @@ -594,7 +220,7 @@ inline float TSDFVolumeCPU::interpolateVoxel(Point3f p) const #if USE_INTRINSICS //gradientDeltaFactor is fixed at 1.0 of voxel size -inline Point3f TSDFVolumeCPU::getNormalVoxel(Point3f _p) const +inline Point3f TSDFVolumeCPU::getNormalVoxel(const Point3f& _p) const { v_float32x4 p(_p.x, _p.y, _p.z, 0.f); v_float32x4 result = getNormalVoxel(p); @@ -662,7 +288,7 @@ inline v_float32x4 TSDFVolumeCPU::getNormalVoxel(const v_float32x4& p) const return Norm.get0() < 0.0001f ? nanv : n/Norm; } #else -inline Point3f TSDFVolumeCPU::getNormalVoxel(Point3f p) const +inline Point3f TSDFVolumeCPU::getNormalVoxel(const Point3f& p) const { const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; const TsdfVoxel* volData = volume.ptr(); @@ -994,8 +620,8 @@ struct RaycastInvoker : ParallelLoopBody }; -void TSDFVolumeCPU::raycast(const cv::Matx44f& cameraPose, const Intr& intrinsics, Size frameSize, - cv::OutputArray _points, cv::OutputArray _normals) const +void TSDFVolumeCPU::raycast(const Matx44f& cameraPose, const Intr& intrinsics, const Size& frameSize, + OutputArray _points, OutputArray _normals) const { CV_TRACE_FUNCTION(); @@ -1189,7 +815,7 @@ void TSDFVolumeCPU::fetchNormals(InputArray _points, OutputArray _normals) const ///////// GPU implementation ///////// #ifdef HAVE_OPENCL -TSDFVolumeGPU::TSDFVolumeGPU(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, +TSDFVolumeGPU::TSDFVolumeGPU(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, Point3i _resolution) : TSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _resolution, false) { @@ -1251,24 +877,25 @@ static cv::UMat preCalculationPixNormGPU(int depth_rows, int depth_cols, Vec2f f // use depth instead of distance (optimization) void TSDFVolumeGPU::integrate(InputArray _depth, float depthFactor, - const cv::Matx44f& cameraPose, const Intr& intrinsics) + const Matx44f& cameraPose, const Intr& intrinsics, const int frameId) { CV_TRACE_FUNCTION(); + CV_UNUSED(frameId); CV_Assert(!_depth.empty()); UMat depth = _depth.getUMat(); - cv::String errorStr; - cv::String name = "integrate"; + String errorStr; + String name = "integrate"; ocl::ProgramSource source = ocl::rgbd::tsdf_oclsrc; - cv::String options = "-cl-mad-enable"; + String options = "-cl-mad-enable"; ocl::Kernel k; k.create(name.c_str(), source, options, &errorStr); if(k.empty()) throw std::runtime_error("Failed to create kernel: " + errorStr); - cv::Affine3f vol2cam(Affine3f(cameraPose.inv()) * pose); + Affine3f vol2cam(Affine3f(cameraPose.inv()) * pose); float dfac = 1.f/depthFactor; Vec4i volResGpu(volResolution.x, volResolution.y, volResolution.z); Vec2f fxy(intrinsics.fx, intrinsics.fy), cxy(intrinsics.cx, intrinsics.cy); @@ -1308,17 +935,17 @@ void TSDFVolumeGPU::integrate(InputArray _depth, float depthFactor, } -void TSDFVolumeGPU::raycast(const cv::Matx44f& cameraPose, const Intr& intrinsics, Size frameSize, - cv::OutputArray _points, cv::OutputArray _normals) const +void TSDFVolumeGPU::raycast(const Matx44f& cameraPose, const Intr& intrinsics, const Size& frameSize, + OutputArray _points, OutputArray _normals) const { CV_TRACE_FUNCTION(); CV_Assert(frameSize.area() > 0); - cv::String errorStr; - cv::String name = "raycast"; + String errorStr; + String name = "raycast"; ocl::ProgramSource source = ocl::rgbd::tsdf_oclsrc; - cv::String options = "-cl-mad-enable"; + String options = "-cl-mad-enable"; ocl::Kernel k; k.create(name.c_str(), source, options, &errorStr); @@ -1384,10 +1011,10 @@ void TSDFVolumeGPU::fetchNormals(InputArray _points, OutputArray _normals) const _normals.createSameSize(_points, POINT_TYPE); UMat normals = _normals.getUMat(); - cv::String errorStr; - cv::String name = "getNormals"; + String errorStr; + String name = "getNormals"; ocl::ProgramSource source = ocl::rgbd::tsdf_oclsrc; - cv::String options = "-cl-mad-enable"; + String options = "-cl-mad-enable"; ocl::Kernel k; k.create(name.c_str(), source, options, &errorStr); @@ -1432,9 +1059,9 @@ void TSDFVolumeGPU::fetchPointsNormals(OutputArray points, OutputArray normals) ocl::Kernel kscan; - cv::String errorStr; + String errorStr; ocl::ProgramSource source = ocl::rgbd::tsdf_oclsrc; - cv::String options = "-cl-mad-enable"; + String options = "-cl-mad-enable"; kscan.create("scanSize", source, options, &errorStr); @@ -1485,7 +1112,7 @@ void TSDFVolumeGPU::fetchPointsNormals(OutputArray points, OutputArray normals) throw std::runtime_error("Failed to run kernel"); Mat groupedSumCpu = groupedSum.getMat(ACCESS_READ); - int gpuSum = (int)cv::sum(groupedSumCpu)[0]; + int gpuSum = (int)sum(groupedSumCpu)[0]; // should be no CPU copies when new kernel is executing groupedSumCpu.release(); @@ -1541,16 +1168,28 @@ void TSDFVolumeGPU::fetchPointsNormals(OutputArray points, OutputArray normals) #endif -cv::Ptr makeTSDFVolume(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, +Ptr makeTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, Point3i _resolution) { #ifdef HAVE_OPENCL - if (cv::ocl::useOpenCL()) - return cv::makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, - _resolution); -#endif - return cv::makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, + if (ocl::useOpenCL()) + return makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _resolution); +#endif + return makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, + _resolution); +} + +Ptr makeTSDFVolume(const VolumeParams& _params) +{ +#ifdef HAVE_OPENCL + if (ocl::useOpenCL()) + return makePtr(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, + _params.tsdfTruncDist, _params.maxWeight, _params.resolution); +#endif + return makePtr(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, + _params.tsdfTruncDist, _params.maxWeight, _params.resolution); + } } // namespace kinfu diff --git a/modules/rgbd/src/tsdf.hpp b/modules/rgbd/src/tsdf.hpp index a4017c73a37..009c4a8466c 100644 --- a/modules/rgbd/src/tsdf.hpp +++ b/modules/rgbd/src/tsdf.hpp @@ -2,7 +2,8 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html -// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this +// module's directory #ifndef __OPENCV_KINFU_TSDF_H__ #define __OPENCV_KINFU_TSDF_H__ @@ -35,7 +36,7 @@ class TSDFVolume : public Volume { public: // dimension in voxels, size in meters - TSDFVolume(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, float _truncDist, + TSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, Point3i _resolution, bool zFirstMemOrder = true); virtual ~TSDFVolume() = default; @@ -57,25 +58,25 @@ class TSDFVolumeCPU : public TSDFVolume TSDFVolumeCPU(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, Vec3i _resolution, bool zFirstMemOrder = true); - virtual void integrate(InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, - const cv::kinfu::Intr& intrinsics) override; - virtual void raycast(const cv::Matx44f& cameraPose, const cv::kinfu::Intr& intrinsics, - cv::Size frameSize, cv::OutputArray points, - cv::OutputArray normals) const override; + virtual void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, + const kinfu::Intr& intrinsics, const int frameId = 0) override; + virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, + OutputArray points, OutputArray normals) const override; - virtual void fetchNormals(cv::InputArray points, cv::OutputArray _normals) const override; - virtual void fetchPointsNormals(cv::OutputArray points, cv::OutputArray normals) const override; + virtual void fetchNormals(InputArray points, OutputArray _normals) const override; + virtual void fetchPointsNormals(OutputArray points, OutputArray normals) const override; virtual void reset() override; - virtual TsdfVoxel at(const cv::Vec3i& volumeIdx) const; + virtual TsdfVoxel at(const Vec3i& volumeIdx) const; - float interpolateVoxel(cv::Point3f p) const; - Point3f getNormalVoxel(cv::Point3f p) const; + float interpolateVoxel(const cv::Point3f& p) const; + Point3f getNormalVoxel(const cv::Point3f& p) const; #if USE_INTRINSICS float interpolateVoxel(const v_float32x4& p) const; v_float32x4 getNormalVoxel(const v_float32x4& p) const; #endif + Vec4i volStrides; Vec6f frameParams; Mat pixNorms; // See zFirstMemOrder arg of parent class constructor @@ -89,17 +90,16 @@ class TSDFVolumeGPU : public TSDFVolume { public: // dimension in voxels, size in meters - TSDFVolumeGPU(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, float _truncDist, + TSDFVolumeGPU(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, Point3i _resolution); - virtual void integrate(InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, - const cv::kinfu::Intr& intrinsics) override; - virtual void raycast(const cv::Matx44f& cameraPose, const cv::kinfu::Intr& intrinsics, - cv::Size frameSize, cv::OutputArray _points, - cv::OutputArray _normals) const override; + virtual void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, + const kinfu::Intr& intrinsics, const int frameId = 0) override; + virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, + OutputArray _points, OutputArray _normals) const override; - virtual void fetchPointsNormals(cv::OutputArray points, cv::OutputArray normals) const override; - virtual void fetchNormals(cv::InputArray points, cv::OutputArray normals) const override; + virtual void fetchPointsNormals(OutputArray points, OutputArray normals) const override; + virtual void fetchNormals(InputArray points, OutputArray normals) const override; virtual void reset() override; @@ -112,8 +112,9 @@ class TSDFVolumeGPU : public TSDFVolume UMat volume; }; #endif -cv::Ptr makeTSDFVolume(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, - float _truncDist, int _maxWeight, Point3i _resolution); +Ptr makeTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, + float _truncDist, int _maxWeight, Point3i _resolution); +Ptr makeTSDFVolume(const VolumeParams& _params); } // namespace kinfu } // namespace cv #endif diff --git a/modules/rgbd/src/tsdf_functions.cpp b/modules/rgbd/src/tsdf_functions.cpp new file mode 100644 index 00000000000..ff40dce248d --- /dev/null +++ b/modules/rgbd/src/tsdf_functions.cpp @@ -0,0 +1,377 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#include "precomp.hpp" +#include "tsdf_functions.hpp" + +namespace cv { + +namespace kinfu { + +cv::Mat preCalculationPixNorm(Depth depth, const Intr& intrinsics) +{ + int height = depth.rows; + int widht = depth.cols; + Point2f fl(intrinsics.fx, intrinsics.fy); + Point2f pp(intrinsics.cx, intrinsics.cy); + Mat pixNorm(height, widht, CV_32F); + std::vector x(widht); + std::vector y(height); + for (int i = 0; i < widht; i++) + x[i] = (i - pp.x) / fl.x; + for (int i = 0; i < height; i++) + y[i] = (i - pp.y) / fl.y; + + for (int i = 0; i < height; i++) + { + for (int j = 0; j < widht; j++) + { + pixNorm.at(i, j) = sqrtf(x[j] * x[j] + y[i] * y[i] + 1.0f); + } + } + return pixNorm; +} + +const bool fixMissingData = false; +depthType bilinearDepth(const Depth& m, cv::Point2f pt) +{ + const depthType defaultValue = qnan; + if (pt.x < 0 || pt.x >= m.cols - 1 || + pt.y < 0 || pt.y >= m.rows - 1) + return defaultValue; + + int xi = cvFloor(pt.x), yi = cvFloor(pt.y); + + const depthType* row0 = m[yi + 0]; + const depthType* row1 = m[yi + 1]; + + depthType v00 = row0[xi + 0]; + depthType v01 = row0[xi + 1]; + depthType v10 = row1[xi + 0]; + depthType v11 = row1[xi + 1]; + + // assume correct depth is positive + bool b00 = v00 > 0; + bool b01 = v01 > 0; + bool b10 = v10 > 0; + bool b11 = v11 > 0; + + if (!fixMissingData) + { + if (!(b00 && b01 && b10 && b11)) + return defaultValue; + else + { + float tx = pt.x - xi, ty = pt.y - yi; + depthType v0 = v00 + tx * (v01 - v00); + depthType v1 = v10 + tx * (v11 - v10); + return v0 + ty * (v1 - v0); + } + } + else + { + int nz = b00 + b01 + b10 + b11; + if (nz == 0) + { + return defaultValue; + } + if (nz == 1) + { + if (b00) return v00; + if (b01) return v01; + if (b10) return v10; + if (b11) return v11; + } + if (nz == 2) + { + if (b00 && b10) v01 = v00, v11 = v10; + if (b01 && b11) v00 = v01, v10 = v11; + if (b00 && b01) v10 = v00, v11 = v01; + if (b10 && b11) v00 = v10, v01 = v11; + if (b00 && b11) v01 = v10 = (v00 + v11) * 0.5f; + if (b01 && b10) v00 = v11 = (v01 + v10) * 0.5f; + } + if (nz == 3) + { + if (!b00) v00 = v10 + v01 - v11; + if (!b01) v01 = v00 + v11 - v10; + if (!b10) v10 = v00 + v11 - v01; + if (!b11) v11 = v01 + v10 - v00; + } + + float tx = pt.x - xi, ty = pt.y - yi; + depthType v0 = v00 + tx * (v01 - v00); + depthType v1 = v10 + tx * (v11 - v10); + return v0 + ty * (v1 - v0); + } +} + +void integrateVolumeUnit( + float truncDist, float voxelSize, int maxWeight, + cv::Matx44f _pose, Point3i volResolution, Vec4i volStrides, + InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, + const cv::kinfu::Intr& intrinsics, InputArray _pixNorms, InputArray _volume) +{ + CV_TRACE_FUNCTION(); + + CV_Assert(_depth.type() == DEPTH_TYPE); + CV_Assert(!_depth.empty()); + cv::Affine3f vpose(_pose); + Depth depth = _depth.getMat(); + + Range integrateRange(0, volResolution.x); + + Mat volume = _volume.getMat(); + Mat pixNorms = _pixNorms.getMat(); + const Intr::Projector proj(intrinsics.makeProjector()); + const cv::Affine3f vol2cam(Affine3f(cameraPose.inv()) * vpose); + const float truncDistInv(1.f / truncDist); + const float dfac(1.f / depthFactor); + TsdfVoxel* volDataStart = volume.ptr();; + +#if USE_INTRINSICS + auto IntegrateInvoker = [&](const Range& range) + { + // zStep == vol2cam*(Point3f(x, y, 1)*voxelSize) - basePt; + Point3f zStepPt = Point3f(vol2cam.matrix(0, 2), + vol2cam.matrix(1, 2), + vol2cam.matrix(2, 2)) * voxelSize; + + v_float32x4 zStep(zStepPt.x, zStepPt.y, zStepPt.z, 0); + v_float32x4 vfxy(proj.fx, proj.fy, 0.f, 0.f), vcxy(proj.cx, proj.cy, 0.f, 0.f); + const v_float32x4 upLimits = v_cvt_f32(v_int32x4(depth.cols - 1, depth.rows - 1, 0, 0)); + + for (int x = range.start; x < range.end; x++) + { + TsdfVoxel* volDataX = volDataStart + x * volStrides[0]; + for (int y = 0; y < volResolution.y; y++) + { + TsdfVoxel* volDataY = volDataX + y * volStrides[1]; + // optimization of camSpace transformation (vector addition instead of matmul at each z) + Point3f basePt = vol2cam * (Point3f((float)x, (float)y, 0) * voxelSize); + v_float32x4 camSpacePt(basePt.x, basePt.y, basePt.z, 0); + + int startZ, endZ; + if (abs(zStepPt.z) > 1e-5) + { + int baseZ = (int)(-basePt.z / zStepPt.z); + if (zStepPt.z > 0) + { + startZ = baseZ; + endZ = volResolution.z; + } + else + { + startZ = 0; + endZ = baseZ; + } + } + else + { + if (basePt.z > 0) + { + startZ = 0; + endZ = volResolution.z; + } + else + { + // z loop shouldn't be performed + startZ = endZ = 0; + } + } + startZ = max(0, startZ); + endZ = min(int(volResolution.z), endZ); + for (int z = startZ; z < endZ; z++) + { + // optimization of the following: + //Point3f volPt = Point3f(x, y, z)*voxelSize; + //Point3f camSpacePt = vol2cam * volPt; + camSpacePt += zStep; + + float zCamSpace = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))).get0(); + if (zCamSpace <= 0.f) + continue; + + v_float32x4 camPixVec = camSpacePt / v_setall_f32(zCamSpace); + v_float32x4 projected = v_muladd(camPixVec, vfxy, vcxy); + // leave only first 2 lanes + projected = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) & + v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)); + + depthType v; + // bilinearly interpolate depth at projected + { + const v_float32x4& pt = projected; + // check coords >= 0 and < imgSize + v_uint32x4 limits = v_reinterpret_as_u32(pt < v_setzero_f32()) | + v_reinterpret_as_u32(pt >= upLimits); + limits = limits | v_rotate_right<1>(limits); + if (limits.get0()) + continue; + + // xi, yi = floor(pt) + v_int32x4 ip = v_floor(pt); + v_int32x4 ipshift = ip; + int xi = ipshift.get0(); + ipshift = v_rotate_right<1>(ipshift); + int yi = ipshift.get0(); + + const depthType* row0 = depth[yi + 0]; + const depthType* row1 = depth[yi + 1]; + + // v001 = [v(xi + 0, yi + 0), v(xi + 1, yi + 0)] + v_float32x4 v001 = v_load_low(row0 + xi); + // v101 = [v(xi + 0, yi + 1), v(xi + 1, yi + 1)] + v_float32x4 v101 = v_load_low(row1 + xi); + + v_float32x4 vall = v_combine_low(v001, v101); + + // assume correct depth is positive + // don't fix missing data + if (v_check_all(vall > v_setzero_f32())) + { + v_float32x4 t = pt - v_cvt_f32(ip); + float tx = t.get0(); + t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); + v_float32x4 ty = v_setall_f32(t.get0()); + // vx is y-interpolated between rows 0 and 1 + v_float32x4 vx = v001 + ty * (v101 - v001); + float v0 = vx.get0(); + vx = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vx))); + float v1 = vx.get0(); + v = v0 + tx * (v1 - v0); + } + else + continue; + } + + // norm(camPixVec) produces double which is too slow + int _u = (int)projected.get0(); + int _v = (int)v_rotate_right<1>(projected).get0(); + if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows)) + continue; + float pixNorm = pixNorms.at(_v, _u); + // float pixNorm = sqrt(v_reduce_sum(camPixVec*camPixVec)); + // difference between distances of point and of surface to camera + float sdf = pixNorm * (v * dfac - zCamSpace); + // possible alternative is: + // kftype sdf = norm(camSpacePt)*(v*dfac/camSpacePt.z - 1); + if (sdf >= -truncDist) + { + TsdfType tsdf = floatToTsdf(fmin(1.f, sdf * truncDistInv)); + + TsdfVoxel& voxel = volDataY[z * volStrides[2]]; + WeightType& weight = voxel.weight; + TsdfType& value = voxel.tsdf; + + // update TSDF + value = floatToTsdf((tsdfToFloat(value) * weight + tsdfToFloat(tsdf)) / (weight + 1)); + weight = (weight + 1) < maxWeight ? (weight + 1) : (WeightType) maxWeight; + } + } + } + } + }; +#else + auto IntegrateInvoker = [&](const Range& range) + { + for (int x = range.start; x < range.end; x++) + { + TsdfVoxel* volDataX = volDataStart + x * volStrides[0]; + for (int y = 0; y < volResolution.y; y++) + { + TsdfVoxel* volDataY = volDataX + y * volStrides[1]; + // optimization of camSpace transformation (vector addition instead of matmul at each z) + Point3f basePt = vol2cam * (Point3f(float(x), float(y), 0.0f) * voxelSize); + Point3f camSpacePt = basePt; + // zStep == vol2cam*(Point3f(x, y, 1)*voxelSize) - basePt; + // zStep == vol2cam*[Point3f(x, y, 1) - Point3f(x, y, 0)]*voxelSize + Point3f zStep = Point3f(vol2cam.matrix(0, 2), + vol2cam.matrix(1, 2), + vol2cam.matrix(2, 2)) * voxelSize; + int startZ, endZ; + if (abs(zStep.z) > 1e-5) + { + int baseZ = int(-basePt.z / zStep.z); + if (zStep.z > 0) + { + startZ = baseZ; + endZ = volResolution.z; + } + else + { + startZ = 0; + endZ = baseZ; + } + } + else + { + if (basePt.z > 0) + { + startZ = 0; + endZ = volResolution.z; + } + else + { + // z loop shouldn't be performed + startZ = endZ = 0; + } + } + startZ = max(0, startZ); + endZ = min(int(volResolution.z), endZ); + + for (int z = startZ; z < endZ; z++) + { + // optimization of the following: + //Point3f volPt = Point3f(x, y, z)*volume.voxelSize; + //Point3f camSpacePt = vol2cam * volPt; + + camSpacePt += zStep; + if (camSpacePt.z <= 0) + continue; + + Point3f camPixVec; + Point2f projected = proj(camSpacePt, camPixVec); + + depthType v = bilinearDepth(depth, projected); + if (v == 0) { + continue; + } + + int _u = projected.x; + int _v = projected.y; + if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows)) + continue; + float pixNorm = pixNorms.at(_v, _u); + + // difference between distances of point and of surface to camera + float sdf = pixNorm * (v * dfac - camSpacePt.z); + // possible alternative is: + // kftype sdf = norm(camSpacePt)*(v*dfac/camSpacePt.z - 1); + if (sdf >= -truncDist) + { + TsdfType tsdf = floatToTsdf(fmin(1.f, sdf * truncDistInv)); + + TsdfVoxel& voxel = volDataY[z * volStrides[2]]; + WeightType& weight = voxel.weight; + TsdfType& value = voxel.tsdf; + + // update TSDF + value = floatToTsdf((tsdfToFloat(value) * weight + tsdfToFloat(tsdf)) / (weight + 1)); + weight = min(int(weight + 1), int(maxWeight)); + } + } + } + } + }; +#endif + + parallel_for_(integrateRange, IntegrateInvoker); + +} + +} // namespace kinfu +} // namespace cv diff --git a/modules/rgbd/src/tsdf_functions.hpp b/modules/rgbd/src/tsdf_functions.hpp new file mode 100644 index 00000000000..6d86595118f --- /dev/null +++ b/modules/rgbd/src/tsdf_functions.hpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#ifndef __OPENCV_TSDF_FUNCTIONS_H__ +#define __OPENCV_TSDF_FUNCTIONS_H__ + +#include +#include "tsdf.hpp" + +namespace cv +{ +namespace kinfu +{ + +inline v_float32x4 tsdfToFloat_INTR(const v_int32x4& num) +{ + v_float32x4 num128 = v_setall_f32(-1.f / 128.f); + return v_cvt_f32(num) * num128; +} + +inline TsdfType floatToTsdf(float num) +{ + //CV_Assert(-1 < num <= 1); + int8_t res = int8_t(num * (-128.f)); + res = res ? res : (num < 0 ? 1 : -1); + return res; +} + +inline float tsdfToFloat(TsdfType num) +{ + return float(num) * (-1.f / 128.f); +} + +cv::Mat preCalculationPixNorm(Depth depth, const Intr& intrinsics); +depthType bilinearDepth(const Depth& m, cv::Point2f pt); + +void integrateVolumeUnit( + float truncDist, float voxelSize, int maxWeight, + cv::Matx44f _pose, Point3i volResolution, Vec4i volStrides, + InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, + const cv::kinfu::Intr& intrinsics, InputArray _pixNorms, InputArray _volume); + +} // namespace kinfu +} // namespace cv +#endif diff --git a/modules/rgbd/src/utils.cpp b/modules/rgbd/src/utils.cpp index 30c8aa153f2..b91a88d4df1 100644 --- a/modules/rgbd/src/utils.cpp +++ b/modules/rgbd/src/utils.cpp @@ -22,7 +22,7 @@ namespace rgbd * @param out_out The rescaled float depth image */ void - rescaleDepth(InputArray in_in, int depth, OutputArray out_out) + rescaleDepth(InputArray in_in, int depth, OutputArray out_out, double depth_factor) { cv::Mat in = in_in.getMat(); CV_Assert(in.type() == CV_64FC1 || in.type() == CV_32FC1 || in.type() == CV_16UC1 || in.type() == CV_16SC1); @@ -34,13 +34,13 @@ namespace rgbd cv::Mat out = out_out.getMat(); if (in_depth == CV_16U) { - in.convertTo(out, depth, 1 / 1000.0); //convert to float so that it is in meters + in.convertTo(out, depth, 1 / depth_factor); //convert to float so that it is in meters cv::Mat valid_mask = in == std::numeric_limits::min(); // Should we do std::numeric_limits::max() too ? out.setTo(std::numeric_limits::quiet_NaN(), valid_mask); //set a$ } if (in_depth == CV_16S) { - in.convertTo(out, depth, 1 / 1000.0); //convert to float so tha$ + in.convertTo(out, depth, 1 / depth_factor); //convert to float so tha$ cv::Mat valid_mask = (in == std::numeric_limits::min()) | (in == std::numeric_limits::max()); // Should we do std::numeric_limits::max() too ? out.setTo(std::numeric_limits::quiet_NaN(), valid_mask); //set a$ } diff --git a/modules/rgbd/src/utils.hpp b/modules/rgbd/src/utils.hpp index b7febed57a7..0b963675390 100644 --- a/modules/rgbd/src/utils.hpp +++ b/modules/rgbd/src/utils.hpp @@ -9,6 +9,8 @@ #ifndef __OPENCV_RGBD_UTILS_HPP__ #define __OPENCV_RGBD_UTILS_HPP__ +#include "precomp.hpp" + namespace cv { namespace rgbd diff --git a/modules/rgbd/src/volume.cpp b/modules/rgbd/src/volume.cpp index 5d83ad5c1a8..8177213e8ab 100644 --- a/modules/rgbd/src/volume.cpp +++ b/modules/rgbd/src/volume.cpp @@ -2,19 +2,79 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html -#include "precomp.hpp" #include -#include "tsdf.hpp" #include "hash_tsdf.hpp" +#include "opencv2/core/base.hpp" +#include "precomp.hpp" +#include "tsdf.hpp" namespace cv { namespace kinfu { -cv::Ptr makeVolume(VolumeType _volumeType, float _voxelSize, cv::Matx44f _pose, - float _raycastStepFactor, float _truncDist, int _maxWeight, - float _truncateThreshold, Vec3i _resolution) +Ptr VolumeParams::defaultParams(VolumeType _volumeType) +{ + VolumeParams params; + params.type = _volumeType; + params.maxWeight = 64; + params.raycastStepFactor = 0.25f; + params.unitResolution = 0; // unitResolution not used for TSDF + float volumeSize = 3.0f; + params.pose = Affine3f().translate(Vec3f(-volumeSize / 2.f, -volumeSize / 2.f, 0.5f)); + if(params.type == VolumeType::TSDF) + { + params.resolution = Vec3i::all(512); + params.voxelSize = volumeSize / 512.f; + params.depthTruncThreshold = 0.f; // depthTruncThreshold not required for TSDF + params.tsdfTruncDist = 7 * params.voxelSize; //! About 0.04f in meters + return makePtr(params); + } + else if(params.type == VolumeType::HASHTSDF) + { + params.unitResolution = 16; + params.voxelSize = volumeSize / 512.f; + params.depthTruncThreshold = rgbd::Odometry::DEFAULT_MAX_DEPTH(); + params.tsdfTruncDist = 7 * params.voxelSize; //! About 0.04f in meters + return makePtr(params); + } + CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); +} + +Ptr VolumeParams::coarseParams(VolumeType _volumeType) +{ + Ptr params = defaultParams(_volumeType); + + params->raycastStepFactor = 0.75f; + float volumeSize = 3.0f; + if(params->type == VolumeType::TSDF) + { + params->resolution = Vec3i::all(128); + params->voxelSize = volumeSize / 128.f; + params->tsdfTruncDist = 2 * params->voxelSize; //! About 0.04f in meters + return params; + } + else if(params->type == VolumeType::HASHTSDF) + { + params->voxelSize = volumeSize / 128.f; + params->tsdfTruncDist = 2 * params->voxelSize; //! About 0.04f in meters + return params; + } + CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); +} + +Ptr makeVolume(const VolumeParams& _volumeParams) +{ + if(_volumeParams.type == VolumeType::TSDF) + return kinfu::makeTSDFVolume(_volumeParams); + else if(_volumeParams.type == VolumeType::HASHTSDF) + return kinfu::makeHashTSDFVolume(_volumeParams); + CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); +} + +Ptr makeVolume(VolumeType _volumeType, float _voxelSize, Matx44f _pose, + float _raycastStepFactor, float _truncDist, int _maxWeight, + float _truncateThreshold, Vec3i _resolution) { Point3i _presolution = _resolution; if (_volumeType == VolumeType::TSDF) @@ -24,11 +84,10 @@ cv::Ptr makeVolume(VolumeType _volumeType, float _voxelSize, cv::Matx44f } else if (_volumeType == VolumeType::HASHTSDF) { - return makeHashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, - _truncateThreshold); + return makeHashTSDFVolume( + _voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold); } - else - return nullptr; + CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); } } // namespace kinfu diff --git a/modules/rgbd/test/test_dynafu.cpp b/modules/rgbd/test/test_dynafu.cpp index 78a1a8fd92f..1ee102e6e0f 100644 --- a/modules/rgbd/test/test_dynafu.cpp +++ b/modules/rgbd/test/test_dynafu.cpp @@ -41,11 +41,11 @@ static const bool display = false; void flyTest(bool hiDense, bool inequal) { - Ptr params; + Ptr params; if(hiDense) - params = dynafu::Params::defaultParams(); + params = kinfu::Params::defaultParams(); else - params = dynafu::Params::coarseParams(); + params = kinfu::Params::coarseParams(); if(inequal) { diff --git a/modules/sfm/CMakeLists.txt b/modules/sfm/CMakeLists.txt index 53a8d4378a9..045a1fe6e45 100644 --- a/modules/sfm/CMakeLists.txt +++ b/modules/sfm/CMakeLists.txt @@ -3,23 +3,56 @@ set(the_description "SFM algorithms") ### LIBMV LIGHT EXTERNAL DEPENDENCIES ### -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") -find_package(Gflags QUIET) + find_package(Ceres QUIET) -if(NOT Ceres_FOUND) # Looks like Ceres find glog on the own, so separate search isn't necessary + +if(NOT Gflags_FOUND) # Ceres find gflags on the own, so separate search isn't necessary + find_package(Gflags QUIET) +endif() +if(NOT Glog_FOUND) # Ceres find glog on the own, so separate search isn't necessary find_package(Glog QUIET) endif() -if((gflags_FOUND OR GFLAGS_FOUND OR GFLAGS_INCLUDE_DIRS) AND (glog_FOUND OR GLOG_FOUND OR GLOG_INCLUDE_DIRS)) - set(_fname "${CMAKE_CURRENT_BINARY_DIR}/test_sfm_deps.cpp") - file(WRITE "${_fname}" "#include \n#include \nint main() { (void)(0); return 0; }\n") - try_compile(SFM_DEPS_OK "${CMAKE_BINARY_DIR}" "${_fname}" - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${GLOG_INCLUDE_DIRS};${GFLAGS_INCLUDE_DIRS}" - LINK_LIBRARIES ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES} - OUTPUT_VARIABLE OUTPUT - ) - file(REMOVE "${_fname}") - message(STATUS "Checking SFM deps... ${SFM_DEPS_OK}") +if(NOT Gflags_FOUND OR NOT Glog_FOUND) + # try local search scripts + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") + if(NOT Gflags_FOUND) + find_package(Gflags QUIET) + endif() + if(NOT Glog_FOUND) + find_package(Glog QUIET) + endif() +endif() + +if(NOT DEFINED GFLAGS_INCLUDE_DIRS AND DEFINED GFLAGS_INCLUDE_DIR) + set(GFLAGS_INCLUDE_DIRS "${GFLAGS_INCLUDE_DIR}") +endif() +if(NOT DEFINED GLOG_INCLUDE_DIRS AND DEFINED GLOG_INCLUDE_DIR) + set(GLOG_INCLUDE_DIRS "${GLOG_INCLUDE_DIR}") +endif() + +if((gflags_FOUND OR Gflags_FOUND OR GFLAGS_FOUND OR GFLAGS_INCLUDE_DIRS) AND (glog_FOUND OR Glog_FOUND OR GLOG_FOUND OR GLOG_INCLUDE_DIRS)) + set(__cache_key "${GLOG_INCLUDE_DIRS} ~ ${GFLAGS_INCLUDE_DIRS} ~ ${GLOG_LIBRARIES} ~ ${GFLAGS_LIBRARIES}") + if(NOT DEFINED SFM_GLOG_GFLAGS_TEST_CACHE_KEY OR NOT (SFM_GLOG_GFLAGS_TEST_CACHE_KEY STREQUAL __cache_key)) + set(__fname "${CMAKE_CURRENT_LIST_DIR}/cmake/checks/check_glog_gflags.cpp") + try_compile( + SFM_GLOG_GFLAGS_TEST "${CMAKE_BINARY_DIR}" "${__fname}" + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${GLOG_INCLUDE_DIRS};${GFLAGS_INCLUDE_DIRS}" + LINK_LIBRARIES ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES} + OUTPUT_VARIABLE __output + ) + if(NOT SFM_GLOG_GFLAGS_TEST) + file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Failed compilation check: ${__fname}\n" + "${__output}\n\n" + ) + endif() + set(SFM_GLOG_GFLAGS_TEST "${SFM_GLOG_GFLAGS_TEST}" CACHE INTERNAL "") + set(SFM_GLOG_GFLAGS_TEST_CACHE_KEY "${__cache_key}" CACHE INTERNAL "") + message(STATUS "Checking SFM glog/gflags deps... ${SFM_GLOG_GFLAGS_TEST}") + endif() + unset(__cache_key) + set(SFM_DEPS_OK "${SFM_GLOG_GFLAGS_TEST}") else() set(SFM_DEPS_OK FALSE) endif() @@ -57,23 +90,14 @@ set(LIBMV_LIGHT_LIBS if(Ceres_FOUND) add_definitions("-DCERES_FOUND=1") list(APPEND LIBMV_LIGHT_LIBS simple_pipeline) - list(APPEND LIBMV_LIGHT_INCLUDES "${CERES_INCLUDE_DIR}") + if(Ceres_VERSION VERSION_LESS 2.0.0) + list(APPEND LIBMV_LIGHT_INCLUDES "${CERES_INCLUDE_DIRS}") + endif() else() add_definitions("-DCERES_FOUND=0") message(STATUS "CERES support is disabled. Ceres Solver for reconstruction API is required.") endif() -### COMPILE WITH C++11 IF CERES WAS COMPILED WITH C++11 - -if(Ceres_FOUND) - list (FIND CERES_COMPILED_COMPONENTS "C++11" _index) - if (${_index} GREATER -1) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") - endif() -endif() - -### DEFINE OPENCV SFM MODULE DEPENDENCIES ### - ### CREATE OPENCV SFM MODULE ### ocv_add_module(sfm @@ -85,6 +109,7 @@ ocv_add_module(sfm WRAP python ) +add_definitions(/DGLOG_NO_ABBREVIATED_SEVERITIES) # avoid ERROR macro conflict in glog (ceres dependency) ocv_warnings_disable(CMAKE_CXX_FLAGS -Wundef @@ -97,12 +122,6 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS -Wsuggest-override ) -if(UNIX) - if(CMAKE_COMPILER_IS_GNUCXX OR CV_ICC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") - endif() -endif() - ocv_include_directories( ${LIBMV_LIGHT_INCLUDES} ) ocv_module_include_directories() @@ -117,14 +136,16 @@ ocv_set_module_sources(HEADERS ${OPENCV_SFM_HDRS} ocv_create_module() -# build libmv_light + +### BUILD libmv_light ### + if(NOT CMAKE_VERSION VERSION_LESS 2.8.11) # See ocv_target_include_directories() implementation if(TARGET ${the_module}) get_target_property(__include_dirs ${the_module} INCLUDE_DIRECTORIES) include_directories(${__include_dirs}) endif() endif() -include_directories(${OCV_TARGET_INCLUDE_DIRS_${the_module}}) +#include_directories(${OCV_TARGET_INCLUDE_DIRS_${the_module}}) add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/src/libmv_light" "${CMAKE_CURRENT_BINARY_DIR}/src/libmv") ocv_target_link_libraries(${the_module} ${LIBMV_LIGHT_LIBS}) @@ -133,6 +154,9 @@ ocv_target_link_libraries(${the_module} ${LIBMV_LIGHT_LIBS}) ### CREATE OPENCV SFM TESTS ### ocv_add_accuracy_tests() +if(Ceres_FOUND AND TARGET opencv_test_sfm) + ocv_target_link_libraries(opencv_test_sfm ${CERES_LIBRARIES}) +endif () ### CREATE OPENCV SFM SAMPLES ### diff --git a/modules/sfm/cmake/checks/check_glog_gflags.cpp b/modules/sfm/cmake/checks/check_glog_gflags.cpp new file mode 100644 index 00000000000..3353419cb3e --- /dev/null +++ b/modules/sfm/cmake/checks/check_glog_gflags.cpp @@ -0,0 +1,7 @@ +#include +#include +int main() +{ + (void)(0); + return 0; +} diff --git a/modules/sfm/src/libmv_capi.h b/modules/sfm/src/libmv_capi.h index d5dd7b9f9f9..20ce63bdc41 100644 --- a/modules/sfm/src/libmv_capi.h +++ b/modules/sfm/src/libmv_capi.h @@ -42,6 +42,8 @@ #ifndef __OPENCV_SFM_LIBMV_CAPI__ #define __OPENCV_SFM_LIBMV_CAPI__ +#include + #include "libmv/logging/logging.h" #include "libmv/correspondence/feature.h" @@ -75,7 +77,7 @@ struct libmv_Reconstruction { EuclideanReconstruction reconstruction; /* Used for per-track average error calculation after reconstruction */ Tracks tracks; - CameraIntrinsics *intrinsics; + std::shared_ptr intrinsics; double error; bool is_valid; }; @@ -252,21 +254,22 @@ static void libmv_cameraIntrinsicsFillFromOptions( * options values. */ -static CameraIntrinsics* libmv_cameraIntrinsicsCreateFromOptions( +static +std::shared_ptr libmv_cameraIntrinsicsCreateFromOptions( const libmv_CameraIntrinsicsOptions* camera_intrinsics_options) { - CameraIntrinsics *camera_intrinsics = NULL; + std::shared_ptr camera_intrinsics; switch (camera_intrinsics_options->distortion_model) { case SFM_DISTORTION_MODEL_POLYNOMIAL: - camera_intrinsics = new PolynomialCameraIntrinsics(); + camera_intrinsics = std::make_shared(); break; case SFM_DISTORTION_MODEL_DIVISION: - camera_intrinsics = new DivisionCameraIntrinsics(); + camera_intrinsics = std::make_shared(); break; default: assert(!"Unknown distortion model"); } libmv_cameraIntrinsicsFillFromOptions(camera_intrinsics_options, - camera_intrinsics); + camera_intrinsics.get()); return camera_intrinsics; } @@ -361,19 +364,19 @@ static void finishReconstruction( /* Perform the complete reconstruction process */ -static libmv_Reconstruction *libmv_solveReconstruction( +static +std::shared_ptr libmv_solveReconstruction( const Tracks &libmv_tracks, const libmv_CameraIntrinsicsOptions* libmv_camera_intrinsics_options, libmv_ReconstructionOptions* libmv_reconstruction_options) { - libmv_Reconstruction *libmv_reconstruction = - new libmv_Reconstruction(); + std::shared_ptr libmv_reconstruction = std::make_shared(); Tracks tracks = libmv_tracks; EuclideanReconstruction &reconstruction = libmv_reconstruction->reconstruction; /* Retrieve reconstruction options from C-API to libmv API. */ - CameraIntrinsics *camera_intrinsics; + std::shared_ptr camera_intrinsics; camera_intrinsics = libmv_reconstruction->intrinsics = libmv_cameraIntrinsicsCreateFromOptions(libmv_camera_intrinsics_options); @@ -426,7 +429,7 @@ static libmv_Reconstruction *libmv_solveReconstruction( libmv_reconstruction_options->refine_intrinsics, libmv::BUNDLE_NO_CONSTRAINTS, &reconstruction, - camera_intrinsics); + camera_intrinsics.get()); } /* Set reconstruction scale to unity. */ @@ -434,10 +437,10 @@ static libmv_Reconstruction *libmv_solveReconstruction( finishReconstruction(tracks, *camera_intrinsics, - libmv_reconstruction); + libmv_reconstruction.get()); libmv_reconstruction->is_valid = true; - return (libmv_Reconstruction *) libmv_reconstruction; + return libmv_reconstruction; } #endif diff --git a/modules/sfm/src/libmv_light/CMakeLists.txt b/modules/sfm/src/libmv_light/CMakeLists.txt index aee38c68184..0a1549ab986 100644 --- a/modules/sfm/src/libmv_light/CMakeLists.txt +++ b/modules/sfm/src/libmv_light/CMakeLists.txt @@ -7,7 +7,7 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS -Winconsistent-missing-override -Wsuggest-o if(CV_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0) ocv_warnings_disable(CMAKE_CXX_FLAGS -Wclass-memaccess) endif() -if(CV_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0) +if((CV_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0) OR CV_CLANG) ocv_warnings_disable(CMAKE_CXX_FLAGS -Wdeprecated-copy) endif() diff --git a/modules/sfm/src/libmv_light/libmv/base/vector.h b/modules/sfm/src/libmv_light/libmv/base/vector.h index 1931fb0b1f9..9740cfaf6fc 100644 --- a/modules/sfm/src/libmv_light/libmv/base/vector.h +++ b/modules/sfm/src/libmv_light/libmv/base/vector.h @@ -121,7 +121,14 @@ class vector { void reserve(unsigned int size) { if (size > size_) { T *data = static_cast(allocate(size)); +#if defined(__GNUC__) && __GNUC__ < 5 // legacy compilers branch memcpy(data, data_, sizeof(*data)*size_); +#else + for (int i = 0; i < size_; ++i) + new (&data[i]) T(std::move(data_[i])); + for (int i = 0; i < size_; ++i) + data_[i].~T(); +#endif allocator_.deallocate(data_, capacity_); data_ = data; capacity_ = size; diff --git a/modules/sfm/src/libmv_light/libmv/multiview/CMakeLists.txt b/modules/sfm/src/libmv_light/libmv/multiview/CMakeLists.txt index 5b4b40b95e2..14c77fe5787 100644 --- a/modules/sfm/src/libmv_light/libmv/multiview/CMakeLists.txt +++ b/modules/sfm/src/libmv_light/libmv/multiview/CMakeLists.txt @@ -21,5 +21,8 @@ TARGET_LINK_LIBRARIES(multiview LINK_PRIVATE ${GLOG_LIBRARY} numeric) IF(TARGET Eigen3::Eigen) TARGET_LINK_LIBRARIES(multiview LINK_PUBLIC Eigen3::Eigen) ENDIF() +IF(CERES_LIBRARIES) + TARGET_LINK_LIBRARIES(multiview LINK_PRIVATE ${CERES_LIBRARIES}) +ENDIF() LIBMV_INSTALL_LIB(multiview) diff --git a/modules/sfm/src/libmv_light/libmv/multiview/fundamental.cc b/modules/sfm/src/libmv_light/libmv/multiview/fundamental.cc index a18dab0ffdf..768c39f5220 100644 --- a/modules/sfm/src/libmv_light/libmv/multiview/fundamental.cc +++ b/modules/sfm/src/libmv_light/libmv/multiview/fundamental.cc @@ -521,7 +521,7 @@ bool EstimateFundamentalFromCorrespondences( FundamentalSymmetricEpipolarCostFunctor, 2, // num_residuals 9>(fundamental_symmetric_epipolar_cost_function), - NULL, + nullptr, F->data()); } diff --git a/modules/sfm/src/libmv_light/libmv/multiview/homography.cc b/modules/sfm/src/libmv_light/libmv/multiview/homography.cc index 8816ef5aa6e..31d99277112 100644 --- a/modules/sfm/src/libmv_light/libmv/multiview/homography.cc +++ b/modules/sfm/src/libmv_light/libmv/multiview/homography.cc @@ -318,7 +318,7 @@ bool EstimateHomography2DFromCorrespondences( HomographySymmetricGeometricCostFunctor, 4, // num_residuals 9>(homography_symmetric_geometric_cost_function), - NULL, + nullptr, H->data()); } diff --git a/modules/sfm/src/libmv_light/libmv/simple_pipeline/bundle.cc b/modules/sfm/src/libmv_light/libmv/simple_pipeline/bundle.cc index 58006e72a2c..1a1568da831 100644 --- a/modules/sfm/src/libmv_light/libmv/simple_pipeline/bundle.cc +++ b/modules/sfm/src/libmv_light/libmv/simple_pipeline/bundle.cc @@ -402,7 +402,7 @@ void EuclideanBundlePointsOnly(const DistortionModelType distortion_model, marker.x, marker.y, 1.0)), - NULL, + nullptr, ceres_intrinsics, current_camera_R_t, &point->X(0)); diff --git a/modules/sfm/src/libmv_light/libmv/simple_pipeline/intersect.cc b/modules/sfm/src/libmv_light/libmv/simple_pipeline/intersect.cc index d625c111951..f13d35ad66d 100644 --- a/modules/sfm/src/libmv_light/libmv/simple_pipeline/intersect.cc +++ b/modules/sfm/src/libmv_light/libmv/simple_pipeline/intersect.cc @@ -113,7 +113,7 @@ bool EuclideanIntersect(const vector &markers, EuclideanIntersectCostFunctor, 2, /* num_residuals */ 3>(new EuclideanIntersectCostFunctor(marker, camera)), - NULL, + nullptr, &X(0)); num_residuals++; } diff --git a/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.cc b/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.cc index d5d009708ba..c0d32bebdce 100644 --- a/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.cc +++ b/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.cc @@ -28,10 +28,6 @@ namespace libmv { -Tracks::Tracks(const Tracks &other) { - markers_ = other.markers_; -} - Tracks::Tracks(const vector &markers) : markers_(markers) {} void Tracks::Insert(int image, int track, double x, double y, double weight) { diff --git a/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.h b/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.h index a54a43659b7..906ec2b6911 100644 --- a/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.h +++ b/modules/sfm/src/libmv_light/libmv/simple_pipeline/tracks.h @@ -65,7 +65,7 @@ class Tracks { Tracks() { } // Copy constructor for a tracks object. - Tracks(const Tracks &other); + Tracks(const Tracks &other) = default; /// Construct a new tracks object using the given markers to start. explicit Tracks(const vector &markers); diff --git a/modules/sfm/src/simple_pipeline.cpp b/modules/sfm/src/simple_pipeline.cpp index a0816108452..476b17e8794 100644 --- a/modules/sfm/src/simple_pipeline.cpp +++ b/modules/sfm/src/simple_pipeline.cpp @@ -118,7 +118,8 @@ parser_2D_tracks( const libmv::Matches &matches, libmv::Tracks &tracks ) * reconstruction pipeline. */ -static libmv_Reconstruction *libmv_solveReconstructionImpl( +static +std::shared_ptr libmv_solveReconstructionImpl( const std::vector &images, const libmv_CameraIntrinsicsOptions* libmv_camera_intrinsics_options, libmv_ReconstructionOptions* libmv_reconstruction_options) @@ -182,9 +183,10 @@ class SFMLibmvReconstructionImpl : public T // Perform reconstruction libmv_reconstruction_ = - *libmv_solveReconstruction(tracks, + libmv_solveReconstruction(tracks, &libmv_camera_intrinsics_options_, &libmv_reconstruction_options_); + CV_Assert(libmv_reconstruction_); } virtual void run(InputArrayOfArrays points2d, InputOutputArray K, OutputArray Rs, @@ -216,9 +218,10 @@ class SFMLibmvReconstructionImpl : public T // Perform reconstruction libmv_reconstruction_ = - *libmv_solveReconstructionImpl(images, + libmv_solveReconstructionImpl(images, &libmv_camera_intrinsics_options_, &libmv_reconstruction_options_); + CV_Assert(libmv_reconstruction_); } @@ -232,12 +235,12 @@ class SFMLibmvReconstructionImpl : public T extractLibmvReconstructionData(K, Rs, Ts, points3d); } - virtual double getError() const { return libmv_reconstruction_.error; } + virtual double getError() const { return libmv_reconstruction_->error; } virtual void getPoints(OutputArray points3d) { const size_t n_points = - libmv_reconstruction_.reconstruction.AllPoints().size(); + libmv_reconstruction_->reconstruction.AllPoints().size(); points3d.create(n_points, 1, CV_64F); @@ -246,7 +249,7 @@ class SFMLibmvReconstructionImpl : public T { for ( int j = 0; j < 3; ++j ) point3d[j] = - libmv_reconstruction_.reconstruction.AllPoints()[i].X[j]; + libmv_reconstruction_->reconstruction.AllPoints()[i].X[j]; Mat(point3d).copyTo(points3d.getMatRef(i)); } @@ -254,14 +257,14 @@ class SFMLibmvReconstructionImpl : public T virtual cv::Mat getIntrinsics() const { Mat K; - eigen2cv(libmv_reconstruction_.intrinsics->K(), K); + eigen2cv(libmv_reconstruction_->intrinsics->K(), K); return K; } virtual void getCameras(OutputArray Rs, OutputArray Ts) { const size_t n_views = - libmv_reconstruction_.reconstruction.AllCameras().size(); + libmv_reconstruction_->reconstruction.AllCameras().size(); Rs.create(n_views, 1, CV_64F); Ts.create(n_views, 1, CV_64F); @@ -270,8 +273,8 @@ class SFMLibmvReconstructionImpl : public T Vec3d t; for(size_t i = 0; i < n_views; ++i) { - eigen2cv(libmv_reconstruction_.reconstruction.AllCameras()[i].R, R); - eigen2cv(libmv_reconstruction_.reconstruction.AllCameras()[i].t, t); + eigen2cv(libmv_reconstruction_->reconstruction.AllCameras()[i].R, R); + eigen2cv(libmv_reconstruction_->reconstruction.AllCameras()[i].t, t); Mat(R).copyTo(Rs.getMatRef(i)); Mat(t).copyTo(Ts.getMatRef(i)); } @@ -300,7 +303,7 @@ class SFMLibmvReconstructionImpl : public T getIntrinsics().copyTo(K.getMat()); } - libmv_Reconstruction libmv_reconstruction_; + std::shared_ptr libmv_reconstruction_; libmv_ReconstructionOptions libmv_reconstruction_options_; libmv_CameraIntrinsicsOptions libmv_camera_intrinsics_options_; }; diff --git a/modules/text/include/opencv2/text/ocr.hpp b/modules/text/include/opencv2/text/ocr.hpp index 0137c37a8b6..6cb23fa4ee4 100644 --- a/modules/text/include/opencv2/text/ocr.hpp +++ b/modules/text/include/opencv2/text/ocr.hpp @@ -474,40 +474,34 @@ class CV_EXPORTS_W OCRBeamSearchDecoder : public BaseOCR @param beam_size Size of the beam in Beam Search algorithm. */ - static Ptr create(const Ptr classifier,// The character classifier with built in feature extractor + static CV_WRAP + Ptr create(const Ptr classifier,// The character classifier with built in feature extractor const std::string& vocabulary, // The language vocabulary (chars when ASCII English text) // size() must be equal to the number of classes InputArray transition_probabilities_table, // Table with transition probabilities between character pairs // cols == rows == vocabulary.size() InputArray emission_probabilities_table, // Table with observation emission probabilities // cols == rows == vocabulary.size() - decoder_mode mode = OCR_DECODER_VITERBI, // HMM Decoding algorithm (only Viterbi for the moment) - int beam_size = 500); // Size of the beam in Beam Search algorithm - - CV_WRAP static Ptr create(const Ptr classifier, // The character classifier with built in feature extractor - const String& vocabulary, // The language vocabulary (chars when ASCII English text) - // size() must be equal to the number of classes - InputArray transition_probabilities_table, // Table with transition probabilities between character pairs - // cols == rows == vocabulary.size() - InputArray emission_probabilities_table, // Table with observation emission probabilities - // cols == rows == vocabulary.size() - int mode = OCR_DECODER_VITERBI, // HMM Decoding algorithm (only Viterbi for the moment) - int beam_size = 500); // Size of the beam in Beam Search algorithm + text::decoder_mode mode = OCR_DECODER_VITERBI, // HMM Decoding algorithm (only Viterbi for the moment) + int beam_size = 500 // Size of the beam in Beam Search algorithm + ); /** @brief Creates an instance of the OCRBeamSearchDecoder class. Initializes HMMDecoder from the specified path. @overload */ - CV_WRAP static Ptr create(const String& filename, // The character classifier file - const String& vocabulary, // The language vocabulary (chars when ASCII English text) + static //CV_WRAP FIXIT bug in handling of Java overloads + Ptr create(const String& filename, // The character classifier file + const String& vocabulary, // The language vocabulary (chars when ASCII English text) // size() must be equal to the number of classes InputArray transition_probabilities_table, // Table with transition probabilities between character pairs // cols == rows == vocabulary.size() InputArray emission_probabilities_table, // Table with observation emission probabilities // cols == rows == vocabulary.size() - int mode = OCR_DECODER_VITERBI, // HMM Decoding algorithm (only Viterbi for the moment) - int beam_size = 500); + text::decoder_mode mode = OCR_DECODER_VITERBI, // HMM Decoding algorithm (only Viterbi for the moment) + int beam_size = 500 // Size of the beam in Beam Search algorithm + ); protected: Ptr classifier; diff --git a/modules/text/src/ocr_beamsearch_decoder.cpp b/modules/text/src/ocr_beamsearch_decoder.cpp index 4b0c43c0fac..6f34056ed90 100644 --- a/modules/text/src/ocr_beamsearch_decoder.cpp +++ b/modules/text/src/ocr_beamsearch_decoder.cpp @@ -499,21 +499,11 @@ Ptr OCRBeamSearchDecoder::create( Ptr(_classifier, _vocabulary, transition_p, emission_p, _mode, _beam_size); } -Ptr OCRBeamSearchDecoder::create(Ptr _classifier, - const String& _vocabulary, - InputArray transition_p, - InputArray emission_p, - int _mode, - int _beam_size) -{ - return makePtr(_classifier, _vocabulary, transition_p, emission_p, (decoder_mode)_mode, _beam_size); -} - Ptr OCRBeamSearchDecoder::create(const String& _filename, const String& _vocabulary, InputArray transition_p, InputArray emission_p, - int _mode, + decoder_mode _mode, int _beam_size) { return makePtr(loadOCRBeamSearchClassifierCNN(_filename), _vocabulary, transition_p, emission_p, (decoder_mode)_mode, _beam_size); diff --git a/modules/tracking/CMakeLists.txt b/modules/tracking/CMakeLists.txt index 055a0593ef8..c638fcf6f4b 100644 --- a/modules/tracking/CMakeLists.txt +++ b/modules/tracking/CMakeLists.txt @@ -1,3 +1,28 @@ set(the_description "Tracking API") -ocv_define_module(tracking opencv_imgproc opencv_core opencv_video opencv_plot OPTIONAL opencv_dnn opencv_datasets WRAP java python objc) + +set(debug_modules "") +if(DEBUG_opencv_tracking) + list(APPEND debug_modules opencv_highgui) +endif() + +ocv_define_module(tracking + opencv_imgproc + opencv_core + opencv_video + opencv_plot # samples only + ${debug_modules} + OPTIONAL + opencv_dnn + opencv_datasets + opencv_highgui + WRAP + java + python + objc +) + ocv_warnings_disable(CMAKE_CXX_FLAGS -Wno-shadow /wd4458) + +if(TARGET opencv_test_${name}) + ocv_target_include_directories(opencv_test_${name} "${OpenCV_SOURCE_DIR}/modules") # use common files from video tests +endif() diff --git a/modules/tracking/doc/tracking.bib b/modules/tracking/doc/tracking.bib index 9853b965f5b..78ce6c32fa1 100644 --- a/modules/tracking/doc/tracking.bib +++ b/modules/tracking/doc/tracking.bib @@ -1,12 +1,3 @@ -@inproceedings{MIL, - title={Visual tracking with online multiple instance learning}, - author={Babenko, Boris and Yang, Ming-Hsuan and Belongie, Serge}, - booktitle={Computer Vision and Pattern Recognition, 2009. CVPR 2009. IEEE Conference on}, - pages={983--990}, - year={2009}, - organization={IEEE} -} - @inproceedings{OLB, title={Real-Time Tracking via On-line Boosting.}, author={Grabner, Helmut and Grabner, Michael and Bischof, Horst}, @@ -37,28 +28,6 @@ @article{TLD publisher={IEEE} } -@article{AAM, - title={Adaptive appearance modeling for video tracking: survey and evaluation}, - author={Salti, Samuele and Cavallaro, Andrea and Di Stefano, Luigi}, - journal={Image Processing, IEEE Transactions on}, - volume={21}, - number={10}, - pages={4334--4348}, - year={2012}, - publisher={IEEE} -} - -@article{AMVOT, - title={A survey of appearance models in visual object tracking}, - author={Li, Xi and Hu, Weiming and Shen, Chunhua and Zhang, Zhongfei and Dick, Anthony and Hengel, Anton Van Den}, - journal={ACM Transactions on Intelligent Systems and Technology (TIST)}, - volume={4}, - number={4}, - pages={58}, - year={2013}, - publisher={ACM} -} - @inproceedings{OOT, title={Online object tracking: A benchmark}, author={Wu, Yi and Lim, Jongwoo and Yang, Ming-Hsuan}, @@ -94,13 +63,6 @@ @INPROCEEDINGS{KCF_CN doi={10.1109/CVPR.2014.143}, } -@inproceedings{GOTURN, -title={Learning to Track at 100 FPS with Deep Regression Networks}, -author={Held, David and Thrun, Sebastian and Savarese, Silvio}, -booktitle = {European Conference Computer Vision (ECCV)}, -year = {2016} -} - @inproceedings{MOSSE, title={Visual Object Tracking using Adaptive Correlation Filters}, author={Bolme, David S. and Beveridge, J. Ross and Draper, Bruce A. and Lui Yui, Man}, diff --git a/modules/tracking/include/opencv2/tracking.hpp b/modules/tracking/include/opencv2/tracking.hpp index 516f5b979bc..f8e73e51f28 100644 --- a/modules/tracking/include/opencv2/tracking.hpp +++ b/modules/tracking/include/opencv2/tracking.hpp @@ -1,284 +1,159 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. -#ifndef __OPENCV_TRACKING_HPP__ -#define __OPENCV_TRACKING_HPP__ +#ifndef OPENCV_CONTRIB_TRACKING_HPP +#define OPENCV_CONTRIB_TRACKING_HPP -#include "opencv2/core/cvdef.h" +#include "opencv2/core.hpp" +#include "opencv2/video/tracking.hpp" -/** @defgroup tracking Tracking API - -Long-term optical tracking API ------------------------------- - -Long-term optical tracking is an important issue for many computer vision applications in -real world scenario. The development in this area is very fragmented and this API is an unique -interface useful for plug several algorithms and compare them. This work is partially based on -@cite AAM and @cite AMVOT . - -These algorithms start from a bounding box of the target and with their internal representation they -avoid the drift during the tracking. These long-term trackers are able to evaluate online the -quality of the location of the target in the new frame, without ground truth. - -There are three main components: the TrackerSampler, the TrackerFeatureSet and the TrackerModel. The -first component is the object that computes the patches over the frame based on the last target -location. The TrackerFeatureSet is the class that manages the Features, is possible plug many kind -of these (HAAR, HOG, LBP, Feature2D, etc). The last component is the internal representation of the -target, it is the appearance model. It stores all state candidates and compute the trajectory (the -most likely target states). The class TrackerTargetState represents a possible state of the target. -The TrackerSampler and the TrackerFeatureSet are the visual representation of the target, instead -the TrackerModel is the statistical model. - -A recent benchmark between these algorithms can be found in @cite OOT - -Creating Your Own %Tracker --------------------- - -If you want to create a new tracker, here's what you have to do. First, decide on the name of the class -for the tracker (to meet the existing style, we suggest something with prefix "tracker", e.g. -trackerMIL, trackerBoosting) -- we shall refer to this choice as to "classname" in subsequent. - -- Declare your tracker in modules/tracking/include/opencv2/tracking/tracker.hpp. Your tracker should inherit from - Tracker (please, see the example below). You should declare the specialized Param structure, - where you probably will want to put the data, needed to initialize your tracker. You should - get something similar to : -@code - class CV_EXPORTS_W TrackerMIL : public Tracker - { - public: - struct CV_EXPORTS Params - { - Params(); - //parameters for sampler - float samplerInitInRadius; // radius for gathering positive instances during init - int samplerInitMaxNegNum; // # negative samples to use during init - float samplerSearchWinSize; // size of search window - float samplerTrackInRadius; // radius for gathering positive instances during tracking - int samplerTrackMaxPosNum; // # positive samples to use during tracking - int samplerTrackMaxNegNum; // # negative samples to use during tracking - int featureSetNumFeatures; // #features - - void read( const FileNode& fn ); - void write( FileStorage& fs ) const; - }; -@endcode - of course, you can also add any additional methods of your choice. It should be pointed out, - however, that it is not expected to have a constructor declared, as creation should be done via - the corresponding create() method. -- Finally, you should implement the function with signature : -@code - Ptr classname::create(const classname::Params ¶meters){ - ... - } -@endcode - That function can (and probably will) return a pointer to some derived class of "classname", - which will probably have a real constructor. - -Every tracker has three component TrackerSampler, TrackerFeatureSet and TrackerModel. The first two -are instantiated from Tracker base class, instead the last component is abstract, so you must -implement your TrackerModel. - -### TrackerSampler - -TrackerSampler is already instantiated, but you should define the sampling algorithm and add the -classes (or single class) to TrackerSampler. You can choose one of the ready implementation as -TrackerSamplerCSC or you can implement your sampling method, in this case the class must inherit -TrackerSamplerAlgorithm. Fill the samplingImpl method that writes the result in "sample" output -argument. - -Example of creating specialized TrackerSamplerAlgorithm TrackerSamplerCSC : : -@code - class CV_EXPORTS_W TrackerSamplerCSC : public TrackerSamplerAlgorithm - { - public: - TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meters = TrackerSamplerCSC::Params() ); - ~TrackerSamplerCSC(); - ... - - protected: - bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ); - ... - - }; -@endcode +namespace cv { +#ifndef CV_DOXYGEN +inline namespace tracking { +#endif -Example of adding TrackerSamplerAlgorithm to TrackerSampler : : -@code - //sampler is the TrackerSampler - Ptr CSCSampler = new TrackerSamplerCSC( CSCparameters ); - if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) - return false; +/** @defgroup tracking Tracking API +@{ + @defgroup tracking_detail Tracking API implementation details + @defgroup tracking_legacy Legacy Tracking API +@} +*/ - //or add CSC sampler with default parameters - //sampler->addTrackerSamplerAlgorithm( "CSC" ); -@endcode -@sa - TrackerSamplerCSC, TrackerSamplerAlgorithm +/** @addtogroup tracking +@{ +Tracking is an important issue for many computer vision applications in real world scenario. +The development in this area is very fragmented and this API is an interface useful for plug several algorithms and compare them. +*/ -### TrackerFeatureSet -TrackerFeatureSet is already instantiated (as first) , but you should define what kinds of features -you'll use in your tracker. You can use multiple feature types, so you can add a ready -implementation as TrackerFeatureHAAR in your TrackerFeatureSet or develop your own implementation. -In this case, in the computeImpl method put the code that extract the features and in the selection -method optionally put the code for the refinement and selection of the features. +/** @brief the CSRT tracker -Example of creating specialized TrackerFeature TrackerFeatureHAAR : : -@code - class CV_EXPORTS_W TrackerFeatureHAAR : public TrackerFeature +The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability +*/ +class CV_EXPORTS_W TrackerCSRT : public Tracker +{ +protected: + TrackerCSRT(); // use ::create() +public: + virtual ~TrackerCSRT() CV_OVERRIDE; + + struct CV_EXPORTS_W_SIMPLE Params { - public: - TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶meters = TrackerFeatureHAAR::Params() ); - ~TrackerFeatureHAAR(); - void selection( Mat& response, int npoints ); - ... - - protected: - bool computeImpl( const std::vector& images, Mat& response ); - ... - + CV_WRAP Params(); + + CV_PROP_RW bool use_hog; + CV_PROP_RW bool use_color_names; + CV_PROP_RW bool use_gray; + CV_PROP_RW bool use_rgb; + CV_PROP_RW bool use_channel_weights; + CV_PROP_RW bool use_segmentation; + + CV_PROP_RW std::string window_function; //!< Window function: "hann", "cheb", "kaiser" + CV_PROP_RW float kaiser_alpha; + CV_PROP_RW float cheb_attenuation; + + CV_PROP_RW float template_size; + CV_PROP_RW float gsl_sigma; + CV_PROP_RW float hog_orientations; + CV_PROP_RW float hog_clip; + CV_PROP_RW float padding; + CV_PROP_RW float filter_lr; + CV_PROP_RW float weights_lr; + CV_PROP_RW int num_hog_channels_used; + CV_PROP_RW int admm_iterations; + CV_PROP_RW int histogram_bins; + CV_PROP_RW float histogram_lr; + CV_PROP_RW int background_ratio; + CV_PROP_RW int number_of_scales; + CV_PROP_RW float scale_sigma_factor; + CV_PROP_RW float scale_model_max_area; + CV_PROP_RW float scale_lr; + CV_PROP_RW float scale_step; + + CV_PROP_RW float psr_threshold; //!< we lost the target, if the psr is lower than this. }; -@endcode -Example of adding TrackerFeature to TrackerFeatureSet : : -@code - //featureSet is the TrackerFeatureSet - Ptr trackerFeature = new TrackerFeatureHAAR( HAARparameters ); - featureSet->addTrackerFeature( trackerFeature ); -@endcode -@sa - TrackerFeatureHAAR, TrackerFeatureSet - -### TrackerModel - -TrackerModel is abstract, so in your implementation you must develop your TrackerModel that inherit -from TrackerModel. Fill the method for the estimation of the state "modelEstimationImpl", that -estimates the most likely target location, see @cite AAM table I (ME) for further information. Fill -"modelUpdateImpl" in order to update the model, see @cite AAM table I (MU). In this class you can use -the :cConfidenceMap and :cTrajectory to storing the model. The first represents the model on the all -possible candidate states and the second represents the list of all estimated states. - -Example of creating specialized TrackerModel TrackerMILModel : : -@code - class TrackerMILModel : public TrackerModel - { - public: - TrackerMILModel( const Rect& boundingBox ); - ~TrackerMILModel(); - ... - - protected: - void modelEstimationImpl( const std::vector& responses ); - void modelUpdateImpl(); - ... + /** @brief Create CSRT tracker instance + @param parameters CSRT parameters TrackerCSRT::Params + */ + static CV_WRAP + Ptr create(const TrackerCSRT::Params ¶meters = TrackerCSRT::Params()); + + //void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; + + CV_WRAP virtual void setInitialMask(InputArray mask) = 0; +}; + + +/** @brief the KCF (Kernelized Correlation Filter) tracker + + * KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. + * This tracking method is an implementation of @cite KCF_ECCV which is extended to KCF with color-names features (@cite KCF_CN). + * The original paper of KCF is available at + * as well as the matlab implementation. For more information about KCF with color-names features, please refer to + * . + */ +class CV_EXPORTS_W TrackerKCF : public Tracker +{ +protected: + TrackerKCF(); // use ::create() +public: + virtual ~TrackerKCF() CV_OVERRIDE; + + /** + * \brief Feature type to be used in the tracking grayscale, colornames, compressed color-names + * The modes available now: + - "GRAY" -- Use grayscale values as the feature + - "CN" -- Color-names feature + */ + enum MODE { + GRAY = (1 << 0), + CN = (1 << 1), + CUSTOM = (1 << 2) }; -@endcode -And add it in your Tracker : : -@code - bool TrackerMIL::initImpl( const Mat& image, const Rect2d& boundingBox ) - { - ... - //model is the general TrackerModel field of the general Tracker - model = new TrackerMILModel( boundingBox ); - ... - } -@endcode -In the last step you should define the TrackerStateEstimator based on your implementation or you can -use one of ready class as TrackerStateEstimatorMILBoosting. It represent the statistical part of the -model that estimates the most likely target state. -Example of creating specialized TrackerStateEstimator TrackerStateEstimatorMILBoosting : : -@code - class CV_EXPORTS_W TrackerStateEstimatorMILBoosting : public TrackerStateEstimator + struct CV_EXPORTS_W_SIMPLE Params { - class TrackerMILTargetState : public TrackerTargetState - { - ... - }; - - public: - TrackerStateEstimatorMILBoosting( int nFeatures = 250 ); - ~TrackerStateEstimatorMILBoosting(); - ... - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ); - void updateImpl( std::vector& confidenceMaps ); - ... - + CV_WRAP Params(); + + CV_PROP_RW float detect_thresh; //!< detection confidence threshold + CV_PROP_RW float sigma; //!< gaussian kernel bandwidth + CV_PROP_RW float lambda; //!< regularization + CV_PROP_RW float interp_factor; //!< linear interpolation factor for adaptation + CV_PROP_RW float output_sigma_factor; //!< spatial bandwidth (proportional to target) + CV_PROP_RW float pca_learning_rate; //!< compression learning rate + CV_PROP_RW bool resize; //!< activate the resize feature to improve the processing speed + CV_PROP_RW bool split_coeff; //!< split the training coefficients into two matrices + CV_PROP_RW bool wrap_kernel; //!< wrap around the kernel values + CV_PROP_RW bool compress_feature; //!< activate the pca method to compress the features + CV_PROP_RW int max_patch_size; //!< threshold for the ROI size + CV_PROP_RW int compressed_size; //!< feature size after compression + CV_PROP_RW int desc_pca; //!< compressed descriptors of TrackerKCF::MODE + CV_PROP_RW int desc_npca; //!< non-compressed descriptors of TrackerKCF::MODE }; -@endcode -And add it in your TrackerModel : : -@code - //model is the TrackerModel of your Tracker - Ptr stateEstimator = new TrackerStateEstimatorMILBoosting( params.featureSetNumFeatures ); - model->setTrackerStateEstimator( stateEstimator ); -@endcode -@sa - TrackerModel, TrackerStateEstimatorMILBoosting, TrackerTargetState -During this step, you should define your TrackerTargetState based on your implementation. -TrackerTargetState base class has only the bounding box (upper-left position, width and height), you -can enrich it adding scale factor, target rotation, etc. + /** @brief Create KCF tracker instance + @param parameters KCF parameters TrackerKCF::Params + */ + static CV_WRAP + Ptr create(const TrackerKCF::Params ¶meters = TrackerKCF::Params()); -Example of creating specialized TrackerTargetState TrackerMILTargetState : : -@code - class TrackerMILTargetState : public TrackerTargetState - { - public: - TrackerMILTargetState( const Point2f& position, int targetWidth, int targetHeight, bool foreground, const Mat& features ); - ~TrackerMILTargetState(); - ... + //void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; - private: - bool isTarget; - Mat targetFeatures; - ... + // FIXIT use interface + typedef void (*FeatureExtractorCallbackFN)(const Mat, const Rect, Mat&); + virtual void setFeatureExtractor(FeatureExtractorCallbackFN callback, bool pca_func = false) = 0; +}; - }; -@endcode -*/ +//! @} -#include -#include +#ifndef CV_DOXYGEN +} +#endif +} // namespace -#endif //__OPENCV_TRACKING_HPP__ +#endif // OPENCV_CONTRIB_TRACKING_HPP diff --git a/modules/tracking/include/opencv2/tracking/feature.hpp b/modules/tracking/include/opencv2/tracking/feature.hpp index 3bcfe6e6c7a..730afd0ebc4 100644 --- a/modules/tracking/include/opencv2/tracking/feature.hpp +++ b/modules/tracking/include/opencv2/tracking/feature.hpp @@ -53,12 +53,15 @@ * TODO Changed CvHaarEvaluator based on ADABOOSTING implementation (Grabner et al.) */ -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { -//! @addtogroup tracking +//! @addtogroup tracking_detail //! @{ +inline namespace contrib_feature { + #define FEATURES "features" #define CC_FEATURES FEATURES @@ -409,8 +412,10 @@ inline uchar CvLBPEvaluator::Feature::calc( const Mat &_sum, size_t y ) const ( psum[p[4]] - psum[p[5]] - psum[p[8]] + psum[p[9]] >= cval ? 1 : 0 ) ); // 3 } +} // namespace + //! @} -} /* namespace cv */ +}}} // namespace cv #endif diff --git a/modules/tracking/include/opencv2/tracking/kalman_filters.hpp b/modules/tracking/include/opencv2/tracking/kalman_filters.hpp index 7a89c87ddf3..0b2e9da053c 100644 --- a/modules/tracking/include/opencv2/tracking/kalman_filters.hpp +++ b/modules/tracking/include/opencv2/tracking/kalman_filters.hpp @@ -45,10 +45,14 @@ #include "opencv2/core.hpp" #include -namespace cv -{ -namespace tracking -{ +namespace cv { +namespace detail { +inline namespace tracking { + +//! @addtogroup tracking_detail +//! @{ + +inline namespace kalman_filters { /** @brief The interface for Unscented Kalman filter and Augmented Unscented Kalman filter. */ @@ -222,7 +226,10 @@ CV_EXPORTS Ptr createUnscentedKalmanFilter( const Unscent */ CV_EXPORTS Ptr createAugmentedUnscentedKalmanFilter( const AugmentedUnscentedKalmanFilterParams ¶ms ); -} // tracking -} // cv +} // namespace + +//! @} + +}}} // namespace #endif diff --git a/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp b/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp index 982bc205b17..a8638c542df 100644 --- a/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp +++ b/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp @@ -44,12 +44,15 @@ #include "opencv2/core.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { -//! @addtogroup tracking +//! @addtogroup tracking_detail //! @{ +inline namespace online_boosting { + //TODO based on the original implementation //http://vision.ucsd.edu/~bbabenko/project_miltrack.shtml @@ -281,8 +284,10 @@ class ClassifierThreshold int m_parity; }; +} // namespace + //! @} -} /* namespace cv */ +}}} // namespace #endif diff --git a/modules/tracking/include/opencv2/tracking/tldDataset.hpp b/modules/tracking/include/opencv2/tracking/tldDataset.hpp index a874255669a..1bdd3fb4c48 100644 --- a/modules/tracking/include/opencv2/tracking/tldDataset.hpp +++ b/modules/tracking/include/opencv2/tracking/tldDataset.hpp @@ -44,13 +44,21 @@ #include "opencv2/core.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { + +//! @addtogroup tracking_detail +//! @{ + namespace tld { CV_EXPORTS cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath = "TLD_dataset", int datasetInd = 0); CV_EXPORTS cv::String tld_getNextDatasetFrame(); } -} + +//! @} + +}}} #endif diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp deleted file mode 100644 index 3c09c2b771b..00000000000 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ /dev/null @@ -1,1548 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#ifndef __OPENCV_TRACKER_HPP__ -#define __OPENCV_TRACKER_HPP__ - -#include "opencv2/core.hpp" -#include "opencv2/imgproc/types_c.h" -#include "feature.hpp" -#include "onlineMIL.hpp" -#include "onlineBoosting.hpp" - -/* - * Partially based on: - * ==================================================================================================================== - * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation - * - [AMVOT] X. Li, W. Hu, C. Shen, Z. Zhang, A. Dick, A. van den Hengel, A Survey of Appearance Models in Visual Object Tracking - * - * This Tracking API has been designed with PlantUML. If you modify this API please change UML files under modules/tracking/doc/uml - * - */ - -namespace cv -{ - -//! @addtogroup tracking -//! @{ - -/************************************ TrackerFeature Base Classes ************************************/ - -/** @brief Abstract base class for TrackerFeature that represents the feature. - */ -class CV_EXPORTS TrackerFeature -{ - public: - virtual ~TrackerFeature(); - - /** @brief Compute the features in the images collection - @param images The images - @param response The output response - */ - void compute( const std::vector& images, Mat& response ); - - /** @brief Create TrackerFeature by tracker feature type - @param trackerFeatureType The TrackerFeature name - - The modes available now: - - - "HAAR" -- Haar Feature-based - - The modes that will be available soon: - - - "HOG" -- Histogram of Oriented Gradients features - - "LBP" -- Local Binary Pattern features - - "FEATURE2D" -- All types of Feature2D - */ - static Ptr create( const String& trackerFeatureType ); - - /** @brief Identify most effective features - @param response Collection of response for the specific TrackerFeature - @param npoints Max number of features - - @note This method modifies the response parameter - */ - virtual void selection( Mat& response, int npoints ) = 0; - - /** @brief Get the name of the specific TrackerFeature - */ - String getClassName() const; - - protected: - - virtual bool computeImpl( const std::vector& images, Mat& response ) = 0; - - String className; -}; - -/** @brief Class that manages the extraction and selection of features - -@cite AAM Feature Extraction and Feature Set Refinement (Feature Processing and Feature Selection). -See table I and section III C @cite AMVOT Appearance modelling -\> Visual representation (Table II, -section 3.1 - 3.2) - -TrackerFeatureSet is an aggregation of TrackerFeature - -@sa - TrackerFeature - - */ -class CV_EXPORTS TrackerFeatureSet -{ - public: - - TrackerFeatureSet(); - - ~TrackerFeatureSet(); - - /** @brief Extract features from the images collection - @param images The input images - */ - void extraction( const std::vector& images ); - - /** @brief Identify most effective features for all feature types (optional) - */ - void selection(); - - /** @brief Remove outliers for all feature types (optional) - */ - void removeOutliers(); - - /** @brief Add TrackerFeature in the collection. Return true if TrackerFeature is added, false otherwise - @param trackerFeatureType The TrackerFeature name - - The modes available now: - - - "HAAR" -- Haar Feature-based - - The modes that will be available soon: - - - "HOG" -- Histogram of Oriented Gradients features - - "LBP" -- Local Binary Pattern features - - "FEATURE2D" -- All types of Feature2D - - Example TrackerFeatureSet::addTrackerFeature : : - @code - //sample usage: - - Ptr trackerFeature = new TrackerFeatureHAAR( HAARparameters ); - featureSet->addTrackerFeature( trackerFeature ); - - //or add CSC sampler with default parameters - //featureSet->addTrackerFeature( "HAAR" ); - @endcode - @note If you use the second method, you must initialize the TrackerFeature - */ - bool addTrackerFeature( String trackerFeatureType ); - - /** @overload - @param feature The TrackerFeature class - */ - bool addTrackerFeature( Ptr& feature ); - - /** @brief Get the TrackerFeature collection (TrackerFeature name, TrackerFeature pointer) - */ - const std::vector > >& getTrackerFeature() const; - - /** @brief Get the responses - - @note Be sure to call extraction before getResponses Example TrackerFeatureSet::getResponses : : - */ - const std::vector& getResponses() const; - - private: - - void clearResponses(); - bool blockAddTrackerFeature; - - std::vector > > features; //list of features - std::vector responses; //list of response after compute - -}; - -/************************************ TrackerSampler Base Classes ************************************/ - -/** @brief Abstract base class for TrackerSamplerAlgorithm that represents the algorithm for the specific -sampler. - */ -class CV_EXPORTS TrackerSamplerAlgorithm -{ - public: - /** - * \brief Destructor - */ - virtual ~TrackerSamplerAlgorithm(); - - /** @brief Create TrackerSamplerAlgorithm by tracker sampler type. - @param trackerSamplerType The trackerSamplerType name - - The modes available now: - - - "CSC" -- Current State Center - - "CS" -- Current State - */ - static Ptr create( const String& trackerSamplerType ); - - /** @brief Computes the regions starting from a position in an image. - - Return true if samples are computed, false otherwise - - @param image The current frame - @param boundingBox The bounding box from which regions can be calculated - - @param sample The computed samples @cite AAM Fig. 1 variable Sk - */ - bool sampling( const Mat& image, Rect boundingBox, std::vector& sample ); - - /** @brief Get the name of the specific TrackerSamplerAlgorithm - */ - String getClassName() const; - - protected: - String className; - - virtual bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) = 0; -}; - -/** - * \brief Class that manages the sampler in order to select regions for the update the model of the tracker - * [AAM] Sampling e Labeling. See table I and section III B - */ - -/** @brief Class that manages the sampler in order to select regions for the update the model of the tracker - -@cite AAM Sampling e Labeling. See table I and section III B - -TrackerSampler is an aggregation of TrackerSamplerAlgorithm -@sa - TrackerSamplerAlgorithm - */ -class CV_EXPORTS TrackerSampler -{ - public: - - /** - * \brief Constructor - */ - TrackerSampler(); - - /** - * \brief Destructor - */ - ~TrackerSampler(); - - /** @brief Computes the regions starting from a position in an image - @param image The current frame - @param boundingBox The bounding box from which regions can be calculated - */ - void sampling( const Mat& image, Rect boundingBox ); - - /** @brief Return the collection of the TrackerSamplerAlgorithm - */ - const std::vector > >& getSamplers() const; - - /** @brief Return the samples from all TrackerSamplerAlgorithm, @cite AAM Fig. 1 variable Sk - */ - const std::vector& getSamples() const; - - /** @brief Add TrackerSamplerAlgorithm in the collection. Return true if sampler is added, false otherwise - @param trackerSamplerAlgorithmType The TrackerSamplerAlgorithm name - - The modes available now: - - "CSC" -- Current State Center - - "CS" -- Current State - - "PF" -- Particle Filtering - - Example TrackerSamplerAlgorithm::addTrackerSamplerAlgorithm : : - @code - TrackerSamplerCSC::Params CSCparameters; - Ptr CSCSampler = new TrackerSamplerCSC( CSCparameters ); - - if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) - return false; - - //or add CSC sampler with default parameters - //sampler->addTrackerSamplerAlgorithm( "CSC" ); - @endcode - @note If you use the second method, you must initialize the TrackerSamplerAlgorithm - */ - bool addTrackerSamplerAlgorithm( String trackerSamplerAlgorithmType ); - - /** @overload - @param sampler The TrackerSamplerAlgorithm - */ - bool addTrackerSamplerAlgorithm( Ptr& sampler ); - - private: - std::vector > > samplers; - std::vector samples; - bool blockAddTrackerSampler; - - void clearSamples(); -}; - -/************************************ TrackerModel Base Classes ************************************/ - -/** @brief Abstract base class for TrackerTargetState that represents a possible state of the target. - -See @cite AAM \f$\hat{x}^{i}_{k}\f$ all the states candidates. - -Inherits this class with your Target state, In own implementation you can add scale variation, -width, height, orientation, etc. - */ -class CV_EXPORTS TrackerTargetState -{ - public: - virtual ~TrackerTargetState() - { - } - ; - /** - * \brief Get the position - * \return The position - */ - Point2f getTargetPosition() const; - - /** - * \brief Set the position - * \param position The position - */ - void setTargetPosition( const Point2f& position ); - /** - * \brief Get the width of the target - * \return The width of the target - */ - int getTargetWidth() const; - - /** - * \brief Set the width of the target - * \param width The width of the target - */ - void setTargetWidth( int width ); - /** - * \brief Get the height of the target - * \return The height of the target - */ - int getTargetHeight() const; - - /** - * \brief Set the height of the target - * \param height The height of the target - */ - void setTargetHeight( int height ); - - protected: - Point2f targetPosition; - int targetWidth; - int targetHeight; - -}; - -/** @brief Represents the model of the target at frame \f$k\f$ (all states and scores) - -See @cite AAM The set of the pair \f$\langle \hat{x}^{i}_{k}, C^{i}_{k} \rangle\f$ -@sa TrackerTargetState - */ -typedef std::vector, float> > ConfidenceMap; - -/** @brief Represents the estimate states for all frames - -@cite AAM \f$x_{k}\f$ is the trajectory of the target up to time \f$k\f$ - -@sa TrackerTargetState - */ -typedef std::vector > Trajectory; - -/** @brief Abstract base class for TrackerStateEstimator that estimates the most likely target state. - -See @cite AAM State estimator - -See @cite AMVOT Statistical modeling (Fig. 3), Table III (generative) - IV (discriminative) - V (hybrid) - */ -class CV_EXPORTS TrackerStateEstimator -{ - public: - virtual ~TrackerStateEstimator(); - - /** @brief Estimate the most likely target state, return the estimated state - @param confidenceMaps The overall appearance model as a list of :cConfidenceMap - */ - Ptr estimate( const std::vector& confidenceMaps ); - - /** @brief Update the ConfidenceMap with the scores - @param confidenceMaps The overall appearance model as a list of :cConfidenceMap - */ - void update( std::vector& confidenceMaps ); - - /** @brief Create TrackerStateEstimator by tracker state estimator type - @param trackeStateEstimatorType The TrackerStateEstimator name - - The modes available now: - - - "BOOSTING" -- Boosting-based discriminative appearance models. See @cite AMVOT section 4.4 - - The modes available soon: - - - "SVM" -- SVM-based discriminative appearance models. See @cite AMVOT section 4.5 - */ - static Ptr create( const String& trackeStateEstimatorType ); - - /** @brief Get the name of the specific TrackerStateEstimator - */ - String getClassName() const; - - protected: - - virtual Ptr estimateImpl( const std::vector& confidenceMaps ) = 0; - virtual void updateImpl( std::vector& confidenceMaps ) = 0; - String className; -}; - -/** @brief Abstract class that represents the model of the target. It must be instantiated by specialized -tracker - -See @cite AAM Ak - -Inherits this with your TrackerModel - */ -class CV_EXPORTS TrackerModel -{ - public: - - /** - * \brief Constructor - */ - TrackerModel(); - - /** - * \brief Destructor - */ - virtual ~TrackerModel(); - - /** @brief Set TrackerEstimator, return true if the tracker state estimator is added, false otherwise - @param trackerStateEstimator The TrackerStateEstimator - @note You can add only one TrackerStateEstimator - */ - bool setTrackerStateEstimator( Ptr trackerStateEstimator ); - - /** @brief Estimate the most likely target location - - @cite AAM ME, Model Estimation table I - @param responses Features extracted from TrackerFeatureSet - */ - void modelEstimation( const std::vector& responses ); - - /** @brief Update the model - - @cite AAM MU, Model Update table I - */ - void modelUpdate(); - - /** @brief Run the TrackerStateEstimator, return true if is possible to estimate a new state, false otherwise - */ - bool runStateEstimator(); - - /** @brief Set the current TrackerTargetState in the Trajectory - @param lastTargetState The current TrackerTargetState - */ - void setLastTargetState( const Ptr& lastTargetState ); - - /** @brief Get the last TrackerTargetState from Trajectory - */ - Ptr getLastTargetState() const; - - /** @brief Get the list of the ConfidenceMap - */ - const std::vector& getConfidenceMaps() const; - - /** @brief Get the last ConfidenceMap for the current frame - */ - const ConfidenceMap& getLastConfidenceMap() const; - - /** @brief Get the TrackerStateEstimator - */ - Ptr getTrackerStateEstimator() const; - - private: - - void clearCurrentConfidenceMap(); - - protected: - std::vector confidenceMaps; - Ptr stateEstimator; - ConfidenceMap currentConfidenceMap; - Trajectory trajectory; - int maxCMLength; - - virtual void modelEstimationImpl( const std::vector& responses ) = 0; - virtual void modelUpdateImpl() = 0; - -}; - -/************************************ Tracker Base Class ************************************/ - -/** @brief Base abstract class for the long-term tracker: - */ -class CV_EXPORTS_W Tracker : public virtual Algorithm -{ - public: - - virtual ~Tracker() CV_OVERRIDE; - - /** @brief Initialize the tracker with a known bounding box that surrounded the target - @param image The initial frame - @param boundingBox The initial bounding box - - @return True if initialization went succesfully, false otherwise - */ - CV_WRAP bool init( InputArray image, const Rect2d& boundingBox ); - - /** @brief Update the tracker, find the new most likely bounding box for the target - @param image The current frame - @param boundingBox The bounding box that represent the new target location, if true was returned, not - modified otherwise - - @return True means that target was located and false means that tracker cannot locate target in - current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed - missing from the frame (say, out of sight) - */ - CV_WRAP bool update( InputArray image, CV_OUT Rect2d& boundingBox ); - - virtual void read( const FileNode& fn ) CV_OVERRIDE = 0; - virtual void write( FileStorage& fs ) const CV_OVERRIDE = 0; - - protected: - - virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; - virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; - - bool isInit; - - Ptr featureSet; - Ptr sampler; - Ptr model; -}; - - -/************************************ Specific TrackerStateEstimator Classes ************************************/ - -/** @brief TrackerStateEstimator based on Boosting - */ -class CV_EXPORTS TrackerStateEstimatorMILBoosting : public TrackerStateEstimator -{ - public: - - /** - * Implementation of the target state for TrackerStateEstimatorMILBoosting - */ - class TrackerMILTargetState : public TrackerTargetState - { - - public: - /** - * \brief Constructor - * \param position Top left corner of the bounding box - * \param width Width of the bounding box - * \param height Height of the bounding box - * \param foreground label for target or background - * \param features features extracted - */ - TrackerMILTargetState( const Point2f& position, int width, int height, bool foreground, const Mat& features ); - - /** - * \brief Destructor - */ - ~TrackerMILTargetState() - { - } - ; - - /** @brief Set label: true for target foreground, false for background - @param foreground Label for background/foreground - */ - void setTargetFg( bool foreground ); - /** @brief Set the features extracted from TrackerFeatureSet - @param features The features extracted - */ - void setFeatures( const Mat& features ); - /** @brief Get the label. Return true for target foreground, false for background - */ - bool isTargetFg() const; - /** @brief Get the features extracted - */ - Mat getFeatures() const; - - private: - bool isTarget; - Mat targetFeatures; - }; - - /** @brief Constructor - @param nFeatures Number of features for each sample - */ - TrackerStateEstimatorMILBoosting( int nFeatures = 250 ); - ~TrackerStateEstimatorMILBoosting(); - - /** @brief Set the current confidenceMap - @param confidenceMap The current :cConfidenceMap - */ - void setCurrentConfidenceMap( ConfidenceMap& confidenceMap ); - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; - void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; - - private: - uint max_idx( const std::vector &v ); - void prepareData( const ConfidenceMap& confidenceMap, Mat& positive, Mat& negative ); - - ClfMilBoost boostMILModel; - bool trained; - int numFeatures; - - ConfidenceMap currentConfidenceMap; -}; - -/** @brief TrackerStateEstimatorAdaBoosting based on ADA-Boosting - */ -class CV_EXPORTS TrackerStateEstimatorAdaBoosting : public TrackerStateEstimator -{ - public: - /** @brief Implementation of the target state for TrackerAdaBoostingTargetState - */ - class TrackerAdaBoostingTargetState : public TrackerTargetState - { - - public: - /** - * \brief Constructor - * \param position Top left corner of the bounding box - * \param width Width of the bounding box - * \param height Height of the bounding box - * \param foreground label for target or background - * \param responses list of features - */ - TrackerAdaBoostingTargetState( const Point2f& position, int width, int height, bool foreground, const Mat& responses ); - - /** - * \brief Destructor - */ - ~TrackerAdaBoostingTargetState() - { - } - ; - - /** @brief Set the features extracted from TrackerFeatureSet - @param responses The features extracted - */ - void setTargetResponses( const Mat& responses ); - /** @brief Set label: true for target foreground, false for background - @param foreground Label for background/foreground - */ - void setTargetFg( bool foreground ); - /** @brief Get the features extracted - */ - Mat getTargetResponses() const; - /** @brief Get the label. Return true for target foreground, false for background - */ - bool isTargetFg() const; - - private: - bool isTarget; - Mat targetResponses; - - }; - - /** @brief Constructor - @param numClassifer Number of base classifiers - @param initIterations Number of iterations in the initialization - @param nFeatures Number of features/weak classifiers - @param patchSize tracking rect - @param ROI initial ROI - */ - TrackerStateEstimatorAdaBoosting( int numClassifer, int initIterations, int nFeatures, Size patchSize, const Rect& ROI ); - - /** - * \brief Destructor - */ - ~TrackerStateEstimatorAdaBoosting(); - - /** @brief Get the sampling ROI - */ - Rect getSampleROI() const; - - /** @brief Set the sampling ROI - @param ROI the sampling ROI - */ - void setSampleROI( const Rect& ROI ); - - /** @brief Set the current confidenceMap - @param confidenceMap The current :cConfidenceMap - */ - void setCurrentConfidenceMap( ConfidenceMap& confidenceMap ); - - /** @brief Get the list of the selected weak classifiers for the classification step - */ - std::vector computeSelectedWeakClassifier(); - - /** @brief Get the list of the weak classifiers that should be replaced - */ - std::vector computeReplacedClassifier(); - - /** @brief Get the list of the weak classifiers that replace those to be replaced - */ - std::vector computeSwappedClassifier(); - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; - void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; - - Ptr boostClassifier; - - private: - int numBaseClassifier; - int iterationInit; - int numFeatures; - bool trained; - Size initPatchSize; - Rect sampleROI; - std::vector replacedClassifier; - std::vector swappedClassifier; - - ConfidenceMap currentConfidenceMap; -}; - -/** - * \brief TrackerStateEstimator based on SVM - */ -class CV_EXPORTS TrackerStateEstimatorSVM : public TrackerStateEstimator -{ - public: - TrackerStateEstimatorSVM(); - ~TrackerStateEstimatorSVM(); - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; - void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; -}; - -/************************************ Specific TrackerSamplerAlgorithm Classes ************************************/ - -/** @brief TrackerSampler based on CSC (current state centered), used by MIL algorithm TrackerMIL - */ -class CV_EXPORTS TrackerSamplerCSC : public TrackerSamplerAlgorithm -{ - public: - enum - { - MODE_INIT_POS = 1, //!< mode for init positive samples - MODE_INIT_NEG = 2, //!< mode for init negative samples - MODE_TRACK_POS = 3, //!< mode for update positive samples - MODE_TRACK_NEG = 4, //!< mode for update negative samples - MODE_DETECT = 5 //!< mode for detect samples - }; - - struct CV_EXPORTS Params - { - Params(); - float initInRad; //!< radius for gathering positive instances during init - float trackInPosRad; //!< radius for gathering positive instances during tracking - float searchWinSize; //!< size of search window - int initMaxNegNum; //!< # negative samples to use during init - int trackMaxPosNum; //!< # positive samples to use during training - int trackMaxNegNum; //!< # negative samples to use during training - }; - - /** @brief Constructor - @param parameters TrackerSamplerCSC parameters TrackerSamplerCSC::Params - */ - TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meters = TrackerSamplerCSC::Params() ); - - /** @brief Set the sampling mode of TrackerSamplerCSC - @param samplingMode The sampling mode - - The modes are: - - - "MODE_INIT_POS = 1" -- for the positive sampling in initialization step - - "MODE_INIT_NEG = 2" -- for the negative sampling in initialization step - - "MODE_TRACK_POS = 3" -- for the positive sampling in update step - - "MODE_TRACK_NEG = 4" -- for the negative sampling in update step - - "MODE_DETECT = 5" -- for the sampling in detection step - */ - void setMode( int samplingMode ); - - ~TrackerSamplerCSC(); - - protected: - - bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) CV_OVERRIDE; - - private: - - Params params; - int mode; - RNG rng; - - std::vector sampleImage( const Mat& img, int x, int y, int w, int h, float inrad, float outrad = 0, int maxnum = 1000000 ); -}; - -/** @brief TrackerSampler based on CS (current state), used by algorithm TrackerBoosting - */ -class CV_EXPORTS TrackerSamplerCS : public TrackerSamplerAlgorithm -{ - public: - enum - { - MODE_POSITIVE = 1, //!< mode for positive samples - MODE_NEGATIVE = 2, //!< mode for negative samples - MODE_CLASSIFY = 3 //!< mode for classify samples - }; - - struct CV_EXPORTS Params - { - Params(); - float overlap; //!& sample ) CV_OVERRIDE; - Rect getROI() const; - private: - Rect getTrackingROI( float searchFactor ); - Rect RectMultiply( const Rect & rect, float f ); - std::vector patchesRegularScan( const Mat& image, Rect trackingROI, Size patchSize ); - void setCheckedROI( Rect imageROI ); - - Params params; - int mode; - Rect trackedPatch; - Rect validROI; - Rect ROI; - -}; - -/** @brief This sampler is based on particle filtering. - -In principle, it can be thought of as performing some sort of optimization (and indeed, this -tracker uses opencv's optim module), where tracker seeks to find the rectangle in given frame, -which is the most *"similar"* to the initial rectangle (the one, given through the constructor). - -The optimization performed is stochastic and somehow resembles genetic algorithms, where on each new -image received (submitted via TrackerSamplerPF::sampling()) we start with the region bounded by -boundingBox, then generate several "perturbed" boxes, take the ones most similar to the original. -This selection round is repeated several times. At the end, we hope that only the most promising box -remaining, and these are combined to produce the subrectangle of image, which is put as a sole -element in array sample. - -It should be noted, that the definition of "similarity" between two rectangles is based on comparing -their histograms. As experiments show, tracker is *not* very succesfull if target is assumed to -strongly change its dimensions. - */ -class CV_EXPORTS TrackerSamplerPF : public TrackerSamplerAlgorithm -{ -public: - /** @brief This structure contains all the parameters that can be varied during the course of sampling - algorithm. Below is the structure exposed, together with its members briefly explained with - reference to the above discussion on algorithm's working. - */ - struct CV_EXPORTS Params - { - Params(); - int iterationNum; //!< number of selection rounds - int particlesNum; //!< number of "perturbed" boxes on each round - double alpha; //!< with each new round we exponentially decrease the amount of "perturbing" we allow (like in simulated annealing) - //!< and this very alpha controls how fast annealing happens, ie. how fast perturbing decreases - Mat_ std; //!< initial values for perturbing (1-by-4 array, as each rectangle is given by 4 values -- coordinates of opposite vertices, - //!< hence we have 4 values to perturb) - }; - /** @brief Constructor - @param chosenRect Initial rectangle, that is supposed to contain target we'd like to track. - @param parameters - */ - TrackerSamplerPF(const Mat& chosenRect,const TrackerSamplerPF::Params ¶meters = TrackerSamplerPF::Params()); -protected: - bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) CV_OVERRIDE; -private: - Params params; - Ptr _solver; - Ptr _function; -}; - -/************************************ Specific TrackerFeature Classes ************************************/ - -/** - * \brief TrackerFeature based on Feature2D - */ -class CV_EXPORTS TrackerFeatureFeature2d : public TrackerFeature -{ - public: - - /** - * \brief Constructor - * \param detectorType string of FeatureDetector - * \param descriptorType string of DescriptorExtractor - */ - TrackerFeatureFeature2d( String detectorType, String descriptorType ); - - ~TrackerFeatureFeature2d() CV_OVERRIDE; - - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - protected: - - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - - private: - - std::vector keypoints; -}; - -/** - * \brief TrackerFeature based on HOG - */ -class CV_EXPORTS TrackerFeatureHOG : public TrackerFeature -{ - public: - - TrackerFeatureHOG(); - - ~TrackerFeatureHOG() CV_OVERRIDE; - - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - protected: - - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - -}; - -/** @brief TrackerFeature based on HAAR features, used by TrackerMIL and many others algorithms -@note HAAR features implementation is copied from apps/traincascade and modified according to MIL - */ -class CV_EXPORTS TrackerFeatureHAAR : public TrackerFeature -{ - public: - struct CV_EXPORTS Params - { - Params(); - int numFeatures; //!< # of rects - Size rectSize; //!< rect size - bool isIntegral; //!< true if input images are integral, false otherwise - }; - - /** @brief Constructor - @param parameters TrackerFeatureHAAR parameters TrackerFeatureHAAR::Params - */ - TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶meters = TrackerFeatureHAAR::Params() ); - - ~TrackerFeatureHAAR() CV_OVERRIDE; - - /** @brief Compute the features only for the selected indices in the images collection - @param selFeatures indices of selected features - @param images The images - @param response Collection of response for the specific TrackerFeature - */ - bool extractSelected( const std::vector selFeatures, const std::vector& images, Mat& response ); - - /** @brief Identify most effective features - @param response Collection of response for the specific TrackerFeature - @param npoints Max number of features - - @note This method modifies the response parameter - */ - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - /** @brief Swap the feature in position source with the feature in position target - @param source The source position - @param target The target position - */ - bool swapFeature( int source, int target ); - - /** @brief Swap the feature in position id with the feature input - @param id The position - @param feature The feature - */ - bool swapFeature( int id, CvHaarEvaluator::FeatureHaar& feature ); - - /** @brief Get the feature in position id - @param id The position - */ - CvHaarEvaluator::FeatureHaar& getFeatureAt( int id ); - - protected: - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - - private: - - Params params; - Ptr featureEvaluator; -}; - -/** - * \brief TrackerFeature based on LBP - */ -class CV_EXPORTS TrackerFeatureLBP : public TrackerFeature -{ - public: - - TrackerFeatureLBP(); - - ~TrackerFeatureLBP(); - - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - protected: - - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - -}; - -/************************************ Specific Tracker Classes ************************************/ - -/** @brief The MIL algorithm trains a classifier in an online manner to separate the object from the -background. - -Multiple Instance Learning avoids the drift problem for a robust tracking. The implementation is -based on @cite MIL . - -Original code can be found here - */ -class CV_EXPORTS_W TrackerMIL : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); - //parameters for sampler - float samplerInitInRadius; //!< radius for gathering positive instances during init - int samplerInitMaxNegNum; //!< # negative samples to use during init - float samplerSearchWinSize; //!< size of search window - float samplerTrackInRadius; //!< radius for gathering positive instances during tracking - int samplerTrackMaxPosNum; //!< # positive samples to use during tracking - int samplerTrackMaxNegNum; //!< # negative samples to use during tracking - int featureSetNumFeatures; //!< # features - - void read( const FileNode& fn ); - void write( FileStorage& fs ) const; - }; - - /** @brief Constructor - @param parameters MIL parameters TrackerMIL::Params - */ - static Ptr create(const TrackerMIL::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerMIL() CV_OVERRIDE {} -}; - -/** @brief the Boosting tracker - -This is a real-time object tracking based on a novel on-line version of the AdaBoost algorithm. -The classifier uses the surrounding background as negative examples in update step to avoid the -drifting problem. The implementation is based on @cite OLB . - */ -class CV_EXPORTS_W TrackerBoosting : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); - int numClassifiers; //! create(const TrackerBoosting::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerBoosting() CV_OVERRIDE {} -}; - -/** @brief the Median Flow tracker - -Implementation of a paper @cite MedianFlow . - -The tracker is suitable for very smooth and predictable movements when object is visible throughout -the whole sequence. It's quite and accurate for this type of problems (in particular, it was shown -by authors to outperform MIL). During the implementation period the code at -, the courtesy of the author Arthur Amarra, was used for the -reference purpose. - */ -class CV_EXPORTS_W TrackerMedianFlow : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); //! create(const TrackerMedianFlow::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerMedianFlow() CV_OVERRIDE {} -}; - -/** @brief the TLD (Tracking, learning and detection) tracker - -TLD is a novel tracking framework that explicitly decomposes the long-term tracking task into -tracking, learning and detection. - -The tracker follows the object from frame to frame. The detector localizes all appearances that -have been observed so far and corrects the tracker if necessary. The learning estimates detector's -errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . - -The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this -implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial -occlusions, object absence etc. - */ -class CV_EXPORTS_W TrackerTLD : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); - void read( const FileNode& /*fn*/ ); - void write( FileStorage& /*fs*/ ) const; - }; - - /** @brief Constructor - @param parameters TLD parameters TrackerTLD::Params - */ - static Ptr create(const TrackerTLD::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerTLD() CV_OVERRIDE {} -}; - -/** @brief the KCF (Kernelized Correlation Filter) tracker - - * KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. - * This tracking method is an implementation of @cite KCF_ECCV which is extended to KCF with color-names features (@cite KCF_CN). - * The original paper of KCF is available at - * as well as the matlab implementation. For more information about KCF with color-names features, please refer to - * . - */ -class CV_EXPORTS_W TrackerKCF : public Tracker -{ -public: - /** - * \brief Feature type to be used in the tracking grayscale, colornames, compressed color-names - * The modes available now: - - "GRAY" -- Use grayscale values as the feature - - "CN" -- Color-names feature - */ - enum MODE { - GRAY = (1 << 0), - CN = (1 << 1), - CUSTOM = (1 << 2) - }; - - struct CV_EXPORTS Params - { - /** - * \brief Constructor - */ - Params(); - - /** - * \brief Read parameters from a file - */ - void read(const FileNode& /*fn*/); - - /** - * \brief Write parameters to a file - */ - void write(FileStorage& /*fs*/) const; - - float detect_thresh; //!< detection confidence threshold - float sigma; //!< gaussian kernel bandwidth - float lambda; //!< regularization - float interp_factor; //!< linear interpolation factor for adaptation - float output_sigma_factor; //!< spatial bandwidth (proportional to target) - float pca_learning_rate; //!< compression learning rate - bool resize; //!< activate the resize feature to improve the processing speed - bool split_coeff; //!< split the training coefficients into two matrices - bool wrap_kernel; //!< wrap around the kernel values - bool compress_feature; //!< activate the pca method to compress the features - int max_patch_size; //!< threshold for the ROI size - int compressed_size; //!< feature size after compression - int desc_pca; //!< compressed descriptors of TrackerKCF::MODE - int desc_npca; //!< non-compressed descriptors of TrackerKCF::MODE - }; - - virtual void setFeatureExtractor(void(*)(const Mat, const Rect, Mat&), bool pca_func = false) = 0; - - /** @brief Constructor - @param parameters KCF parameters TrackerKCF::Params - */ - static Ptr create(const TrackerKCF::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerKCF() CV_OVERRIDE {} -}; - -/** @brief the GOTURN (Generic Object Tracking Using Regression Networks) tracker - - * GOTURN (@cite GOTURN) is kind of trackers based on Convolutional Neural Networks (CNN). While taking all advantages of CNN trackers, - * GOTURN is much faster due to offline training without online fine-tuning nature. - * GOTURN tracker addresses the problem of single target tracking: given a bounding box label of an object in the first frame of the video, - * we track that object through the rest of the video. NOTE: Current method of GOTURN does not handle occlusions; however, it is fairly - * robust to viewpoint changes, lighting changes, and deformations. - * Inputs of GOTURN are two RGB patches representing Target and Search patches resized to 227x227. - * Outputs of GOTURN are predicted bounding box coordinates, relative to Search patch coordinate system, in format X1,Y1,X2,Y2. - * Original paper is here: - * As long as original authors implementation: - * Implementation of training algorithm is placed in separately here due to 3d-party dependencies: - * - * GOTURN architecture goturn.prototxt and trained model goturn.caffemodel are accessible on opencv_extra GitHub repository. -*/ -class CV_EXPORTS_W TrackerGOTURN : public Tracker -{ -public: - struct CV_EXPORTS Params - { - Params(); - void read(const FileNode& /*fn*/); - void write(FileStorage& /*fs*/) const; - String modelTxt; - String modelBin; - }; - - /** @brief Constructor - @param parameters GOTURN parameters TrackerGOTURN::Params - */ - static Ptr create(const TrackerGOTURN::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerGOTURN() CV_OVERRIDE {} -}; - -/** @brief the MOSSE (Minimum Output Sum of Squared %Error) tracker - -The implementation is based on @cite MOSSE Visual Object Tracking using Adaptive Correlation Filters -@note this tracker works with grayscale images, if passed bgr ones, they will get converted internally. -*/ - -class CV_EXPORTS_W TrackerMOSSE : public Tracker -{ - public: - /** @brief Constructor - */ - CV_WRAP static Ptr create(); - - virtual ~TrackerMOSSE() CV_OVERRIDE {} -}; - - -/************************************ MultiTracker Class ---By Laksono Kurnianggoro---) ************************************/ -/** @brief This class is used to track multiple objects using the specified tracker algorithm. - -* The %MultiTracker is naive implementation of multiple object tracking. -* It process the tracked objects independently without any optimization accross the tracked objects. -*/ -class CV_EXPORTS_W MultiTracker : public Algorithm -{ -public: - - /** - * \brief Constructor. - */ - CV_WRAP MultiTracker(); - - /** - * \brief Destructor - */ - ~MultiTracker() CV_OVERRIDE; - - /** - * \brief Add a new object to be tracked. - * - * @param newTracker tracking algorithm to be used - * @param image input image - * @param boundingBox a rectangle represents ROI of the tracked object - */ - CV_WRAP bool add(Ptr newTracker, InputArray image, const Rect2d& boundingBox); - - /** - * \brief Add a set of objects to be tracked. - * @param newTrackers list of tracking algorithms to be used - * @param image input image - * @param boundingBox list of the tracked objects - */ - bool add(std::vector > newTrackers, InputArray image, std::vector boundingBox); - - /** - * \brief Update the current tracking status. - * The result will be saved in the internal storage. - * @param image input image - */ - bool update(InputArray image); - - /** - * \brief Update the current tracking status. - * @param image input image - * @param boundingBox the tracking result, represent a list of ROIs of the tracked objects. - */ - CV_WRAP bool update(InputArray image, CV_OUT std::vector & boundingBox); - - /** - * \brief Returns a reference to a storage for the tracked objects, each object corresponds to one tracker algorithm - */ - CV_WRAP const std::vector& getObjects() const; - - /** - * \brief Returns a pointer to a new instance of MultiTracker - */ - CV_WRAP static Ptr create(); - -protected: - //!< storage for the tracker algorithms. - std::vector< Ptr > trackerList; - - //!< storage for the tracked objects, each object corresponds to one tracker algorithm. - std::vector objects; -}; - -/************************************ Multi-Tracker Classes ---By Tyan Vladimir---************************************/ - -/** @brief Base abstract class for the long-term Multi Object Trackers: - -@sa Tracker, MultiTrackerTLD -*/ -class CV_EXPORTS MultiTracker_Alt -{ -public: - /** @brief Constructor for Multitracker - */ - MultiTracker_Alt() - { - targetNum = 0; - } - - /** @brief Add a new target to a tracking-list and initialize the tracker with a known bounding box that surrounded the target - @param image The initial frame - @param boundingBox The initial bounding box of target - @param tracker_algorithm Multi-tracker algorithm - - @return True if new target initialization went succesfully, false otherwise - */ - bool addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm); - - /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets - @param image The current frame - - @return True means that all targets were located and false means that tracker couldn't locate one of the targets in - current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed - missing from the frame (say, out of sight) - */ - bool update(InputArray image); - - /** @brief Current number of targets in tracking-list - */ - int targetNum; - - /** @brief Trackers list for Multi-Object-Tracker - */ - std::vector > trackers; - - /** @brief Bounding Boxes list for Multi-Object-Tracker - */ - std::vector boundingBoxes; - /** @brief List of randomly generated colors for bounding boxes display - */ - std::vector colors; -}; - -/** @brief Multi Object %Tracker for TLD. - -TLD is a novel tracking framework that explicitly decomposes -the long-term tracking task into tracking, learning and detection. - -The tracker follows the object from frame to frame. The detector localizes all appearances that -have been observed so far and corrects the tracker if necessary. The learning estimates detector's -errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . - -The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this -implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial -occlusions, object absence etc. - -@sa Tracker, MultiTracker, TrackerTLD -*/ -class CV_EXPORTS MultiTrackerTLD : public MultiTracker_Alt -{ -public: - /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets by - optimized update method using some techniques to speedup calculations specifically for MO TLD. The only limitation - is that all target bounding boxes should have approximately same aspect ratios. Speed boost is around 20% - - @param image The current frame. - - @return True means that all targets were located and false means that tracker couldn't locate one of the targets in - current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed - missing from the frame (say, out of sight) - */ - bool update_opt(InputArray image); -}; - -/*********************************** CSRT ************************************/ -/** @brief the CSRT tracker - -The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability -*/ -class CV_EXPORTS_W TrackerCSRT : public Tracker -{ -public: - struct CV_EXPORTS Params - { - /** - * \brief Constructor - */ - Params(); - - /** - * \brief Read parameters from a file - */ - void read(const FileNode& /*fn*/); - - /** - * \brief Write parameters to a file - */ - void write(cv::FileStorage& fs) const; - - bool use_hog; - bool use_color_names; - bool use_gray; - bool use_rgb; - bool use_channel_weights; - bool use_segmentation; - - std::string window_function; //!< Window function: "hann", "cheb", "kaiser" - float kaiser_alpha; - float cheb_attenuation; - - float template_size; - float gsl_sigma; - float hog_orientations; - float hog_clip; - float padding; - float filter_lr; - float weights_lr; - int num_hog_channels_used; - int admm_iterations; - int histogram_bins; - float histogram_lr; - int background_ratio; - int number_of_scales; - float scale_sigma_factor; - float scale_model_max_area; - float scale_lr; - float scale_step; - - float psr_threshold; //!< we lost the target, if the psr is lower than this. - }; - - /** @brief Constructor - @param parameters CSRT parameters TrackerCSRT::Params - */ - static Ptr create(const TrackerCSRT::Params ¶meters); - - CV_WRAP static Ptr create(); - - CV_WRAP virtual void setInitialMask(InputArray mask) = 0; - - virtual ~TrackerCSRT() CV_OVERRIDE {} -}; - -//! @} -} /* namespace cv */ - -#endif diff --git a/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp b/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp index b6962e2a9b1..6590c2a1eef 100644 --- a/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp +++ b/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp @@ -20,6 +20,12 @@ namespace cv { +namespace detail { +inline namespace tracking { + +//! @addtogroup tracking_detail +//! @{ + namespace tbm { //Tracking-by-Matching /// /// \brief The TrackedObject struct defines properties of detected object. @@ -553,5 +559,8 @@ class CV_EXPORTS ITrackerByMatching { CV_EXPORTS cv::Ptr createTrackerByMatching(const TrackerParams ¶ms = TrackerParams()); } // namespace tbm -} // namespace cv + +//! @} + +}}} // namespace #endif // #ifndef __OPENCV_TRACKING_TRACKING_BY_MATCHING_HPP__ diff --git a/modules/tracking/include/opencv2/tracking/tracking_internals.hpp b/modules/tracking/include/opencv2/tracking/tracking_internals.hpp new file mode 100644 index 00000000000..92ca54a992a --- /dev/null +++ b/modules/tracking/include/opencv2/tracking/tracking_internals.hpp @@ -0,0 +1,932 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_TRACKING_DETAIL_HPP +#define OPENCV_TRACKING_DETAIL_HPP + +/* + * Partially based on: + * ==================================================================================================================== + * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation + * - [AMVOT] X. Li, W. Hu, C. Shen, Z. Zhang, A. Dick, A. van den Hengel, A Survey of Appearance Models in Visual Object Tracking + * + * This Tracking API has been designed with PlantUML. If you modify this API please change UML files under modules/tracking/doc/uml + * + */ + +#include "opencv2/video/detail/tracking.private.hpp" + +#include "feature.hpp" // CvHaarEvaluator +#include "onlineBoosting.hpp" // StrongClassifierDirectSelection + +namespace cv { +namespace detail { +inline namespace tracking { + +/** @addtogroup tracking_detail +@{ + +Long-term optical tracking API +------------------------------ + +Long-term optical tracking is an important issue for many computer vision applications in +real world scenario. The development in this area is very fragmented and this API is an unique +interface useful for plug several algorithms and compare them. This work is partially based on +@cite AAM and @cite AMVOT . + +These algorithms start from a bounding box of the target and with their internal representation they +avoid the drift during the tracking. These long-term trackers are able to evaluate online the +quality of the location of the target in the new frame, without ground truth. + +There are three main components: the TrackerContribSampler, the TrackerContribFeatureSet and the TrackerModel. The +first component is the object that computes the patches over the frame based on the last target +location. The TrackerContribFeatureSet is the class that manages the Features, is possible plug many kind +of these (HAAR, HOG, LBP, Feature2D, etc). The last component is the internal representation of the +target, it is the appearance model. It stores all state candidates and compute the trajectory (the +most likely target states). The class TrackerTargetState represents a possible state of the target. +The TrackerContribSampler and the TrackerContribFeatureSet are the visual representation of the target, instead +the TrackerModel is the statistical model. + +A recent benchmark between these algorithms can be found in @cite OOT + +Creating Your Own %Tracker +-------------------- + +If you want to create a new tracker, here's what you have to do. First, decide on the name of the class +for the tracker (to meet the existing style, we suggest something with prefix "tracker", e.g. +trackerMIL, trackerBoosting) -- we shall refer to this choice as to "classname" in subsequent. + +- Declare your tracker in modules/tracking/include/opencv2/tracking/tracker.hpp. Your tracker should inherit from + Tracker (please, see the example below). You should declare the specialized Param structure, + where you probably will want to put the data, needed to initialize your tracker. You should + get something similar to : +@code + class CV_EXPORTS_W TrackerMIL : public Tracker + { + public: + struct CV_EXPORTS Params + { + Params(); + //parameters for sampler + float samplerInitInRadius; // radius for gathering positive instances during init + int samplerInitMaxNegNum; // # negative samples to use during init + float samplerSearchWinSize; // size of search window + float samplerTrackInRadius; // radius for gathering positive instances during tracking + int samplerTrackMaxPosNum; // # positive samples to use during tracking + int samplerTrackMaxNegNum; // # negative samples to use during tracking + int featureSetNumFeatures; // #features + + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + }; +@endcode + of course, you can also add any additional methods of your choice. It should be pointed out, + however, that it is not expected to have a constructor declared, as creation should be done via + the corresponding create() method. +- Finally, you should implement the function with signature : +@code + Ptr classname::create(const classname::Params ¶meters){ + ... + } +@endcode + That function can (and probably will) return a pointer to some derived class of "classname", + which will probably have a real constructor. + +Every tracker has three component TrackerContribSampler, TrackerContribFeatureSet and TrackerModel. The first two +are instantiated from Tracker base class, instead the last component is abstract, so you must +implement your TrackerModel. + +### TrackerContribSampler + +TrackerContribSampler is already instantiated, but you should define the sampling algorithm and add the +classes (or single class) to TrackerContribSampler. You can choose one of the ready implementation as +TrackerContribSamplerCSC or you can implement your sampling method, in this case the class must inherit +TrackerContribSamplerAlgorithm. Fill the samplingImpl method that writes the result in "sample" output +argument. + +Example of creating specialized TrackerContribSamplerAlgorithm TrackerContribSamplerCSC : : +@code + class CV_EXPORTS_W TrackerContribSamplerCSC : public TrackerContribSamplerAlgorithm + { + public: + TrackerContribSamplerCSC( const TrackerContribSamplerCSC::Params ¶meters = TrackerContribSamplerCSC::Params() ); + ~TrackerContribSamplerCSC(); + ... + + protected: + bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ); + ... + + }; +@endcode + +Example of adding TrackerContribSamplerAlgorithm to TrackerContribSampler : : +@code + //sampler is the TrackerContribSampler + Ptr CSCSampler = new TrackerContribSamplerCSC( CSCparameters ); + if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) + return false; + + //or add CSC sampler with default parameters + //sampler->addTrackerSamplerAlgorithm( "CSC" ); +@endcode +@sa + TrackerContribSamplerCSC, TrackerContribSamplerAlgorithm + +### TrackerContribFeatureSet + +TrackerContribFeatureSet is already instantiated (as first) , but you should define what kinds of features +you'll use in your tracker. You can use multiple feature types, so you can add a ready +implementation as TrackerContribFeatureHAAR in your TrackerContribFeatureSet or develop your own implementation. +In this case, in the computeImpl method put the code that extract the features and in the selection +method optionally put the code for the refinement and selection of the features. + +Example of creating specialized TrackerFeature TrackerContribFeatureHAAR : : +@code + class CV_EXPORTS_W TrackerContribFeatureHAAR : public TrackerFeature + { + public: + TrackerContribFeatureHAAR( const TrackerContribFeatureHAAR::Params ¶meters = TrackerContribFeatureHAAR::Params() ); + ~TrackerContribFeatureHAAR(); + void selection( Mat& response, int npoints ); + ... + + protected: + bool computeImpl( const std::vector& images, Mat& response ); + ... + + }; +@endcode +Example of adding TrackerFeature to TrackerContribFeatureSet : : +@code + //featureSet is the TrackerContribFeatureSet + Ptr trackerFeature = new TrackerContribFeatureHAAR( HAARparameters ); + featureSet->addTrackerFeature( trackerFeature ); +@endcode +@sa + TrackerContribFeatureHAAR, TrackerContribFeatureSet + +### TrackerModel + +TrackerModel is abstract, so in your implementation you must develop your TrackerModel that inherit +from TrackerModel. Fill the method for the estimation of the state "modelEstimationImpl", that +estimates the most likely target location, see @cite AAM table I (ME) for further information. Fill +"modelUpdateImpl" in order to update the model, see @cite AAM table I (MU). In this class you can use +the :cConfidenceMap and :cTrajectory to storing the model. The first represents the model on the all +possible candidate states and the second represents the list of all estimated states. + +Example of creating specialized TrackerModel TrackerMILModel : : +@code + class TrackerMILModel : public TrackerModel + { + public: + TrackerMILModel( const Rect& boundingBox ); + ~TrackerMILModel(); + ... + + protected: + void modelEstimationImpl( const std::vector& responses ); + void modelUpdateImpl(); + ... + + }; +@endcode +And add it in your Tracker : : +@code + bool TrackerMIL::initImpl( const Mat& image, const Rect2d& boundingBox ) + { + ... + //model is the general TrackerModel field of the general Tracker + model = new TrackerMILModel( boundingBox ); + ... + } +@endcode +In the last step you should define the TrackerStateEstimator based on your implementation or you can +use one of ready class as TrackerStateEstimatorMILBoosting. It represent the statistical part of the +model that estimates the most likely target state. + +Example of creating specialized TrackerStateEstimator TrackerStateEstimatorMILBoosting : : +@code + class CV_EXPORTS_W TrackerStateEstimatorMILBoosting : public TrackerStateEstimator + { + class TrackerMILTargetState : public TrackerTargetState + { + ... + }; + + public: + TrackerStateEstimatorMILBoosting( int nFeatures = 250 ); + ~TrackerStateEstimatorMILBoosting(); + ... + + protected: + Ptr estimateImpl( const std::vector& confidenceMaps ); + void updateImpl( std::vector& confidenceMaps ); + ... + + }; +@endcode +And add it in your TrackerModel : : +@code + //model is the TrackerModel of your Tracker + Ptr stateEstimator = new TrackerStateEstimatorMILBoosting( params.featureSetNumFeatures ); + model->setTrackerStateEstimator( stateEstimator ); +@endcode +@sa + TrackerModel, TrackerStateEstimatorMILBoosting, TrackerTargetState + +During this step, you should define your TrackerTargetState based on your implementation. +TrackerTargetState base class has only the bounding box (upper-left position, width and height), you +can enrich it adding scale factor, target rotation, etc. + +Example of creating specialized TrackerTargetState TrackerMILTargetState : : +@code + class TrackerMILTargetState : public TrackerTargetState + { + public: + TrackerMILTargetState( const Point2f& position, int targetWidth, int targetHeight, bool foreground, const Mat& features ); + ~TrackerMILTargetState(); + ... + + private: + bool isTarget; + Mat targetFeatures; + ... + + }; +@endcode + +*/ + + +/************************************ TrackerContribFeature Base Classes ************************************/ + +/** @brief Abstract base class for TrackerContribFeature that represents the feature. + */ +class CV_EXPORTS TrackerContribFeature : public TrackerFeature +{ + public: + virtual ~TrackerContribFeature(); + + /** @brief Create TrackerContribFeature by tracker feature type + @param trackerFeatureType The TrackerContribFeature name + + The modes available now: + + - "HAAR" -- Haar Feature-based + + The modes that will be available soon: + + - "HOG" -- Histogram of Oriented Gradients features + - "LBP" -- Local Binary Pattern features + - "FEATURE2D" -- All types of Feature2D + */ + static Ptr create( const String& trackerFeatureType ); + + /** @brief Identify most effective features + @param response Collection of response for the specific TrackerContribFeature + @param npoints Max number of features + + @note This method modifies the response parameter + */ + virtual void selection( Mat& response, int npoints ) = 0; + + /** @brief Get the name of the specific TrackerContribFeature + */ + String getClassName() const; + + protected: + String className; +}; + +/** @brief Class that manages the extraction and selection of features + +@cite AAM Feature Extraction and Feature Set Refinement (Feature Processing and Feature Selection). +See table I and section III C @cite AMVOT Appearance modelling -\> Visual representation (Table II, +section 3.1 - 3.2) + +TrackerContribFeatureSet is an aggregation of TrackerContribFeature + +@sa + TrackerContribFeature + + */ +class CV_EXPORTS TrackerContribFeatureSet +{ + public: + + TrackerContribFeatureSet(); + + ~TrackerContribFeatureSet(); + + /** @brief Extract features from the images collection + @param images The input images + */ + void extraction( const std::vector& images ); + + /** @brief Identify most effective features for all feature types (optional) + */ + void selection(); + + /** @brief Remove outliers for all feature types (optional) + */ + void removeOutliers(); + + /** @brief Add TrackerContribFeature in the collection. Return true if TrackerContribFeature is added, false otherwise + @param trackerFeatureType The TrackerContribFeature name + + The modes available now: + + - "HAAR" -- Haar Feature-based + + The modes that will be available soon: + + - "HOG" -- Histogram of Oriented Gradients features + - "LBP" -- Local Binary Pattern features + - "FEATURE2D" -- All types of Feature2D + + Example TrackerContribFeatureSet::addTrackerFeature : : + @code + //sample usage: + + Ptr trackerFeature = ...; + featureSet->addTrackerFeature( trackerFeature ); + + //or add CSC sampler with default parameters + //featureSet->addTrackerFeature( "HAAR" ); + @endcode + @note If you use the second method, you must initialize the TrackerContribFeature + */ + bool addTrackerFeature( String trackerFeatureType ); + + /** @overload + @param feature The TrackerContribFeature class + */ + bool addTrackerFeature( Ptr& feature ); + + /** @brief Get the TrackerContribFeature collection (TrackerContribFeature name, TrackerContribFeature pointer) + */ + const std::vector > >& getTrackerFeature() const; + + /** @brief Get the responses + + @note Be sure to call extraction before getResponses Example TrackerContribFeatureSet::getResponses : : + */ + const std::vector& getResponses() const; + + private: + + void clearResponses(); + bool blockAddTrackerFeature; + + std::vector > > features; //list of features + std::vector responses; //list of response after compute + +}; + + +/************************************ TrackerContribSampler Base Classes ************************************/ + +/** @brief Abstract base class for TrackerContribSamplerAlgorithm that represents the algorithm for the specific +sampler. + */ +class CV_EXPORTS TrackerContribSamplerAlgorithm : public TrackerSamplerAlgorithm +{ + public: + /** + * \brief Destructor + */ + virtual ~TrackerContribSamplerAlgorithm(); + + /** @brief Create TrackerContribSamplerAlgorithm by tracker sampler type. + @param trackerSamplerType The trackerSamplerType name + + The modes available now: + + - "CSC" -- Current State Center + - "CS" -- Current State + */ + static Ptr create( const String& trackerSamplerType ); + + /** @brief Computes the regions starting from a position in an image. + + Return true if samples are computed, false otherwise + + @param image The current frame + @param boundingBox The bounding box from which regions can be calculated + + @param sample The computed samples @cite AAM Fig. 1 variable Sk + */ + virtual bool sampling(const Mat& image, const Rect& boundingBox, std::vector& sample) CV_OVERRIDE; + + /** @brief Get the name of the specific TrackerContribSamplerAlgorithm + */ + String getClassName() const; + + protected: + String className; + + virtual bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) = 0; +}; + +/** + * \brief Class that manages the sampler in order to select regions for the update the model of the tracker + * [AAM] Sampling e Labeling. See table I and section III B + */ + +/** @brief Class that manages the sampler in order to select regions for the update the model of the tracker + +@cite AAM Sampling e Labeling. See table I and section III B + +TrackerContribSampler is an aggregation of TrackerContribSamplerAlgorithm +@sa + TrackerContribSamplerAlgorithm + */ +class CV_EXPORTS TrackerContribSampler +{ + public: + + /** + * \brief Constructor + */ + TrackerContribSampler(); + + /** + * \brief Destructor + */ + ~TrackerContribSampler(); + + /** @brief Computes the regions starting from a position in an image + @param image The current frame + @param boundingBox The bounding box from which regions can be calculated + */ + void sampling( const Mat& image, Rect boundingBox ); + + /** @brief Return the collection of the TrackerContribSamplerAlgorithm + */ + const std::vector > >& getSamplers() const; + + /** @brief Return the samples from all TrackerContribSamplerAlgorithm, @cite AAM Fig. 1 variable Sk + */ + const std::vector& getSamples() const; + + /** @brief Add TrackerContribSamplerAlgorithm in the collection. Return true if sampler is added, false otherwise + @param trackerSamplerAlgorithmType The TrackerContribSamplerAlgorithm name + + The modes available now: + - "CSC" -- Current State Center + - "CS" -- Current State + - "PF" -- Particle Filtering + + Example TrackerContribSamplerAlgorithm::addTrackerContribSamplerAlgorithm : : + @code + TrackerContribSamplerCSC::Params CSCparameters; + Ptr CSCSampler = new TrackerContribSamplerCSC( CSCparameters ); + + if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) + return false; + + //or add CSC sampler with default parameters + //sampler->addTrackerSamplerAlgorithm( "CSC" ); + @endcode + @note If you use the second method, you must initialize the TrackerContribSamplerAlgorithm + */ + bool addTrackerSamplerAlgorithm( String trackerSamplerAlgorithmType ); + + /** @overload + @param sampler The TrackerContribSamplerAlgorithm + */ + bool addTrackerSamplerAlgorithm( Ptr& sampler ); + + private: + std::vector > > samplers; + std::vector samples; + bool blockAddTrackerSampler; + + void clearSamples(); +}; + + +/** @brief TrackerStateEstimatorAdaBoosting based on ADA-Boosting + */ +class CV_EXPORTS TrackerStateEstimatorAdaBoosting : public TrackerStateEstimator +{ + public: + /** @brief Implementation of the target state for TrackerAdaBoostingTargetState + */ + class CV_EXPORTS TrackerAdaBoostingTargetState : public TrackerTargetState + { + + public: + /** + * \brief Constructor + * \param position Top left corner of the bounding box + * \param width Width of the bounding box + * \param height Height of the bounding box + * \param foreground label for target or background + * \param responses list of features + */ + TrackerAdaBoostingTargetState( const Point2f& position, int width, int height, bool foreground, const Mat& responses ); + + /** + * \brief Destructor + */ + ~TrackerAdaBoostingTargetState() + { + } + ; + + /** @brief Set the features extracted from TrackerContribFeatureSet + @param responses The features extracted + */ + void setTargetResponses( const Mat& responses ); + /** @brief Set label: true for target foreground, false for background + @param foreground Label for background/foreground + */ + void setTargetFg( bool foreground ); + /** @brief Get the features extracted + */ + Mat getTargetResponses() const; + /** @brief Get the label. Return true for target foreground, false for background + */ + bool isTargetFg() const; + + private: + bool isTarget; + Mat targetResponses; + + }; + + /** @brief Constructor + @param numClassifer Number of base classifiers + @param initIterations Number of iterations in the initialization + @param nFeatures Number of features/weak classifiers + @param patchSize tracking rect + @param ROI initial ROI + */ + TrackerStateEstimatorAdaBoosting( int numClassifer, int initIterations, int nFeatures, Size patchSize, const Rect& ROI ); + + /** + * \brief Destructor + */ + ~TrackerStateEstimatorAdaBoosting(); + + /** @brief Get the sampling ROI + */ + Rect getSampleROI() const; + + /** @brief Set the sampling ROI + @param ROI the sampling ROI + */ + void setSampleROI( const Rect& ROI ); + + /** @brief Set the current confidenceMap + @param confidenceMap The current :cConfidenceMap + */ + void setCurrentConfidenceMap( ConfidenceMap& confidenceMap ); + + /** @brief Get the list of the selected weak classifiers for the classification step + */ + std::vector computeSelectedWeakClassifier(); + + /** @brief Get the list of the weak classifiers that should be replaced + */ + std::vector computeReplacedClassifier(); + + /** @brief Get the list of the weak classifiers that replace those to be replaced + */ + std::vector computeSwappedClassifier(); + + protected: + Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; + void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; + + Ptr boostClassifier; + + private: + int numBaseClassifier; + int iterationInit; + int numFeatures; + bool trained; + Size initPatchSize; + Rect sampleROI; + std::vector replacedClassifier; + std::vector swappedClassifier; + + ConfidenceMap currentConfidenceMap; +}; + + +/** + * \brief TrackerStateEstimator based on SVM + */ +class CV_EXPORTS TrackerStateEstimatorSVM : public TrackerStateEstimator +{ + public: + TrackerStateEstimatorSVM(); + ~TrackerStateEstimatorSVM(); + + protected: + Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; + void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; +}; + + + +/************************************ Specific TrackerSamplerAlgorithm Classes ************************************/ + +/** @brief TrackerSampler based on CSC (current state centered), used by MIL algorithm TrackerMIL + */ +class CV_EXPORTS TrackerContribSamplerCSC : public TrackerContribSamplerAlgorithm +{ + public: + enum + { + MODE_INIT_POS = 1, //!< mode for init positive samples + MODE_INIT_NEG = 2, //!< mode for init negative samples + MODE_TRACK_POS = 3, //!< mode for update positive samples + MODE_TRACK_NEG = 4, //!< mode for update negative samples + MODE_DETECT = 5 //!< mode for detect samples + }; + + struct CV_EXPORTS Params + { + Params(); + float initInRad; //!< radius for gathering positive instances during init + float trackInPosRad; //!< radius for gathering positive instances during tracking + float searchWinSize; //!< size of search window + int initMaxNegNum; //!< # negative samples to use during init + int trackMaxPosNum; //!< # positive samples to use during training + int trackMaxNegNum; //!< # negative samples to use during training + }; + + /** @brief Constructor + @param parameters TrackerContribSamplerCSC parameters TrackerContribSamplerCSC::Params + */ + TrackerContribSamplerCSC( const TrackerContribSamplerCSC::Params ¶meters = TrackerContribSamplerCSC::Params() ); + + /** @brief Set the sampling mode of TrackerContribSamplerCSC + @param samplingMode The sampling mode + + The modes are: + + - "MODE_INIT_POS = 1" -- for the positive sampling in initialization step + - "MODE_INIT_NEG = 2" -- for the negative sampling in initialization step + - "MODE_TRACK_POS = 3" -- for the positive sampling in update step + - "MODE_TRACK_NEG = 4" -- for the negative sampling in update step + - "MODE_DETECT = 5" -- for the sampling in detection step + */ + void setMode( int samplingMode ); + + ~TrackerContribSamplerCSC(); + + protected: + + bool samplingImpl(const Mat& image, Rect boundingBox, std::vector& sample) CV_OVERRIDE; + + private: + + Params params; + int mode; + RNG rng; + + std::vector sampleImage( const Mat& img, int x, int y, int w, int h, float inrad, float outrad = 0, int maxnum = 1000000 ); +}; + + +/** @brief TrackerContribSampler based on CS (current state), used by algorithm TrackerBoosting + */ +class CV_EXPORTS TrackerSamplerCS : public TrackerContribSamplerAlgorithm +{ + public: + enum + { + MODE_POSITIVE = 1, //!< mode for positive samples + MODE_NEGATIVE = 2, //!< mode for negative samples + MODE_CLASSIFY = 3 //!< mode for classify samples + }; + + struct CV_EXPORTS Params + { + Params(); + float overlap; //!& sample ) CV_OVERRIDE; + Rect getROI() const; + private: + Rect getTrackingROI( float searchFactor ); + Rect RectMultiply( const Rect & rect, float f ); + std::vector patchesRegularScan( const Mat& image, Rect trackingROI, Size patchSize ); + void setCheckedROI( Rect imageROI ); + + Params params; + int mode; + Rect trackedPatch; + Rect validROI; + Rect ROI; + +}; + +/** @brief This sampler is based on particle filtering. + +In principle, it can be thought of as performing some sort of optimization (and indeed, this +tracker uses opencv's optim module), where tracker seeks to find the rectangle in given frame, +which is the most *"similar"* to the initial rectangle (the one, given through the constructor). + +The optimization performed is stochastic and somehow resembles genetic algorithms, where on each new +image received (submitted via TrackerSamplerPF::sampling()) we start with the region bounded by +boundingBox, then generate several "perturbed" boxes, take the ones most similar to the original. +This selection round is repeated several times. At the end, we hope that only the most promising box +remaining, and these are combined to produce the subrectangle of image, which is put as a sole +element in array sample. + +It should be noted, that the definition of "similarity" between two rectangles is based on comparing +their histograms. As experiments show, tracker is *not* very succesfull if target is assumed to +strongly change its dimensions. + */ +class CV_EXPORTS TrackerSamplerPF : public TrackerContribSamplerAlgorithm +{ +public: + /** @brief This structure contains all the parameters that can be varied during the course of sampling + algorithm. Below is the structure exposed, together with its members briefly explained with + reference to the above discussion on algorithm's working. + */ + struct CV_EXPORTS Params + { + Params(); + int iterationNum; //!< number of selection rounds + int particlesNum; //!< number of "perturbed" boxes on each round + double alpha; //!< with each new round we exponentially decrease the amount of "perturbing" we allow (like in simulated annealing) + //!< and this very alpha controls how fast annealing happens, ie. how fast perturbing decreases + Mat_ std; //!< initial values for perturbing (1-by-4 array, as each rectangle is given by 4 values -- coordinates of opposite vertices, + //!< hence we have 4 values to perturb) + }; + /** @brief Constructor + @param chosenRect Initial rectangle, that is supposed to contain target we'd like to track. + @param parameters + */ + TrackerSamplerPF(const Mat& chosenRect,const TrackerSamplerPF::Params ¶meters = TrackerSamplerPF::Params()); +protected: + bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) CV_OVERRIDE; +private: + Params params; + Ptr _solver; + Ptr _function; +}; + + + +/************************************ Specific TrackerContribFeature Classes ************************************/ + +/** + * \brief TrackerContribFeature based on Feature2D + */ +class CV_EXPORTS TrackerFeatureFeature2d : public TrackerContribFeature +{ + public: + + /** + * \brief Constructor + * \param detectorType string of FeatureDetector + * \param descriptorType string of DescriptorExtractor + */ + TrackerFeatureFeature2d( String detectorType, String descriptorType ); + + ~TrackerFeatureFeature2d() CV_OVERRIDE; + + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + protected: + + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + + private: + + std::vector keypoints; +}; + +/** + * \brief TrackerContribFeature based on HOG + */ +class CV_EXPORTS TrackerFeatureHOG : public TrackerContribFeature +{ + public: + + TrackerFeatureHOG(); + + ~TrackerFeatureHOG() CV_OVERRIDE; + + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + protected: + + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + +}; + +/** @brief TrackerContribFeature based on HAAR features, used by TrackerMIL and many others algorithms +@note HAAR features implementation is copied from apps/traincascade and modified according to MIL + */ +class CV_EXPORTS TrackerContribFeatureHAAR : public TrackerContribFeature +{ + public: + struct CV_EXPORTS Params + { + Params(); + int numFeatures; //!< # of rects + Size rectSize; //!< rect size + bool isIntegral; //!< true if input images are integral, false otherwise + }; + + /** @brief Constructor + @param parameters TrackerContribFeatureHAAR parameters TrackerContribFeatureHAAR::Params + */ + TrackerContribFeatureHAAR( const TrackerContribFeatureHAAR::Params ¶meters = TrackerContribFeatureHAAR::Params() ); + + ~TrackerContribFeatureHAAR() CV_OVERRIDE; + + /** @brief Compute the features only for the selected indices in the images collection + @param selFeatures indices of selected features + @param images The images + @param response Collection of response for the specific TrackerContribFeature + */ + bool extractSelected( const std::vector selFeatures, const std::vector& images, Mat& response ); + + /** @brief Identify most effective features + @param response Collection of response for the specific TrackerContribFeature + @param npoints Max number of features + + @note This method modifies the response parameter + */ + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + /** @brief Swap the feature in position source with the feature in position target + @param source The source position + @param target The target position + */ + bool swapFeature( int source, int target ); + + /** @brief Swap the feature in position id with the feature input + @param id The position + @param feature The feature + */ + bool swapFeature( int id, CvHaarEvaluator::FeatureHaar& feature ); + + /** @brief Get the feature in position id + @param id The position + */ + CvHaarEvaluator::FeatureHaar& getFeatureAt( int id ); + + protected: + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + + private: + + Params params; + Ptr featureEvaluator; +}; + +/** + * \brief TrackerContribFeature based on LBP + */ +class CV_EXPORTS TrackerFeatureLBP : public TrackerContribFeature +{ + public: + + TrackerFeatureLBP(); + + ~TrackerFeatureLBP(); + + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + protected: + + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + +}; + +//! @} + +}}} // namespace + +#endif // OPENCV_TRACKING_DETAIL_HPP diff --git a/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp b/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp new file mode 100644 index 00000000000..e0f17064c62 --- /dev/null +++ b/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp @@ -0,0 +1,538 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#ifndef OPENCV_TRACKING_LEGACY_HPP +#define OPENCV_TRACKING_LEGACY_HPP + +/* + * Partially based on: + * ==================================================================================================================== + * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation + * - [AMVOT] X. Li, W. Hu, C. Shen, Z. Zhang, A. Dick, A. van den Hengel, A Survey of Appearance Models in Visual Object Tracking + * + * This Tracking API has been designed with PlantUML. If you modify this API please change UML files under modules/tracking/doc/uml + * + */ + +#include "tracking_internals.hpp" + +namespace cv { +namespace legacy { +#ifndef CV_DOXYGEN +inline namespace tracking { +#endif +using namespace cv::detail::tracking; + +/** @addtogroup tracking_legacy +@{ +*/ + +/************************************ Tracker Base Class ************************************/ + +/** @brief Base abstract class for the long-term tracker: + */ +class CV_EXPORTS_W Tracker : public virtual Algorithm +{ + public: + Tracker(); + virtual ~Tracker() CV_OVERRIDE; + + /** @brief Initialize the tracker with a known bounding box that surrounded the target + @param image The initial frame + @param boundingBox The initial bounding box + + @return True if initialization went succesfully, false otherwise + */ + CV_WRAP bool init( InputArray image, const Rect2d& boundingBox ); + + /** @brief Update the tracker, find the new most likely bounding box for the target + @param image The current frame + @param boundingBox The bounding box that represent the new target location, if true was returned, not + modified otherwise + + @return True means that target was located and false means that tracker cannot locate target in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ + CV_WRAP bool update( InputArray image, CV_OUT Rect2d& boundingBox ); + + virtual void read( const FileNode& fn ) CV_OVERRIDE = 0; + virtual void write( FileStorage& fs ) const CV_OVERRIDE = 0; + + protected: + + virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; + virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; + + bool isInit; + + Ptr featureSet; + Ptr sampler; + Ptr model; +}; + + +/************************************ Specific Tracker Classes ************************************/ + +/** @brief The MIL algorithm trains a classifier in an online manner to separate the object from the +background. + +Multiple Instance Learning avoids the drift problem for a robust tracking. The implementation is +based on @cite MIL . + +Original code can be found here + */ +class CV_EXPORTS_W TrackerMIL : public cv::legacy::Tracker +{ + public: + struct CV_EXPORTS Params : cv::TrackerMIL::Params + { + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + }; + + /** @brief Constructor + @param parameters MIL parameters TrackerMIL::Params + */ + static Ptr create(const TrackerMIL::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerMIL() CV_OVERRIDE {} +}; + +/** @brief the Boosting tracker + +This is a real-time object tracking based on a novel on-line version of the AdaBoost algorithm. +The classifier uses the surrounding background as negative examples in update step to avoid the +drifting problem. The implementation is based on @cite OLB . + */ +class CV_EXPORTS_W TrackerBoosting : public cv::legacy::Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); + int numClassifiers; //! create(const TrackerBoosting::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerBoosting() CV_OVERRIDE {} +}; + +/** @brief the Median Flow tracker + +Implementation of a paper @cite MedianFlow . + +The tracker is suitable for very smooth and predictable movements when object is visible throughout +the whole sequence. It's quite and accurate for this type of problems (in particular, it was shown +by authors to outperform MIL). During the implementation period the code at +, the courtesy of the author Arthur Amarra, was used for the +reference purpose. + */ +class CV_EXPORTS_W TrackerMedianFlow : public cv::legacy::Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); //! create(const TrackerMedianFlow::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerMedianFlow() CV_OVERRIDE {} +}; + +/** @brief the TLD (Tracking, learning and detection) tracker + +TLD is a novel tracking framework that explicitly decomposes the long-term tracking task into +tracking, learning and detection. + +The tracker follows the object from frame to frame. The detector localizes all appearances that +have been observed so far and corrects the tracker if necessary. The learning estimates detector's +errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . + +The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this +implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial +occlusions, object absence etc. + */ +class CV_EXPORTS_W TrackerTLD : public cv::legacy::Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); + void read( const FileNode& /*fn*/ ); + void write( FileStorage& /*fs*/ ) const; + }; + + /** @brief Constructor + @param parameters TLD parameters TrackerTLD::Params + */ + static Ptr create(const TrackerTLD::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerTLD() CV_OVERRIDE {} +}; + +/** @brief the KCF (Kernelized Correlation Filter) tracker + + * KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. + * This tracking method is an implementation of @cite KCF_ECCV which is extended to KCF with color-names features (@cite KCF_CN). + * The original paper of KCF is available at + * as well as the matlab implementation. For more information about KCF with color-names features, please refer to + * . + */ +class CV_EXPORTS_W TrackerKCF : public cv::legacy::Tracker +{ +public: + /** + * \brief Feature type to be used in the tracking grayscale, colornames, compressed color-names + * The modes available now: + - "GRAY" -- Use grayscale values as the feature + - "CN" -- Color-names feature + */ + typedef enum cv::tracking::TrackerKCF::MODE MODE; + + struct CV_EXPORTS Params : cv::tracking::TrackerKCF::Params + { + void read(const FileNode& /*fn*/); + void write(FileStorage& /*fs*/) const; + }; + + virtual void setFeatureExtractor(void(*)(const Mat, const Rect, Mat&), bool pca_func = false) = 0; + + /** @brief Constructor + @param parameters KCF parameters TrackerKCF::Params + */ + static Ptr create(const TrackerKCF::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerKCF() CV_OVERRIDE {} +}; + +#if 0 // legacy variant is not available +/** @brief the GOTURN (Generic Object Tracking Using Regression Networks) tracker + + * GOTURN (@cite GOTURN) is kind of trackers based on Convolutional Neural Networks (CNN). While taking all advantages of CNN trackers, + * GOTURN is much faster due to offline training without online fine-tuning nature. + * GOTURN tracker addresses the problem of single target tracking: given a bounding box label of an object in the first frame of the video, + * we track that object through the rest of the video. NOTE: Current method of GOTURN does not handle occlusions; however, it is fairly + * robust to viewpoint changes, lighting changes, and deformations. + * Inputs of GOTURN are two RGB patches representing Target and Search patches resized to 227x227. + * Outputs of GOTURN are predicted bounding box coordinates, relative to Search patch coordinate system, in format X1,Y1,X2,Y2. + * Original paper is here: + * As long as original authors implementation: + * Implementation of training algorithm is placed in separately here due to 3d-party dependencies: + * + * GOTURN architecture goturn.prototxt and trained model goturn.caffemodel are accessible on opencv_extra GitHub repository. +*/ +class CV_EXPORTS_W TrackerGOTURN : public cv::legacy::Tracker +{ +public: + struct CV_EXPORTS Params + { + Params(); + void read(const FileNode& /*fn*/); + void write(FileStorage& /*fs*/) const; + String modelTxt; + String modelBin; + }; + + /** @brief Constructor + @param parameters GOTURN parameters TrackerGOTURN::Params + */ + static Ptr create(const TrackerGOTURN::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerGOTURN() CV_OVERRIDE {} +}; +#endif + +/** @brief the MOSSE (Minimum Output Sum of Squared %Error) tracker + +The implementation is based on @cite MOSSE Visual Object Tracking using Adaptive Correlation Filters +@note this tracker works with grayscale images, if passed bgr ones, they will get converted internally. +*/ + +class CV_EXPORTS_W TrackerMOSSE : public cv::legacy::Tracker +{ + public: + /** @brief Constructor + */ + CV_WRAP static Ptr create(); + + virtual ~TrackerMOSSE() CV_OVERRIDE {} +}; + + +/************************************ MultiTracker Class ---By Laksono Kurnianggoro---) ************************************/ +/** @brief This class is used to track multiple objects using the specified tracker algorithm. + +* The %MultiTracker is naive implementation of multiple object tracking. +* It process the tracked objects independently without any optimization accross the tracked objects. +*/ +class CV_EXPORTS_W MultiTracker : public Algorithm +{ +public: + + /** + * \brief Constructor. + */ + CV_WRAP MultiTracker(); + + /** + * \brief Destructor + */ + ~MultiTracker() CV_OVERRIDE; + + /** + * \brief Add a new object to be tracked. + * + * @param newTracker tracking algorithm to be used + * @param image input image + * @param boundingBox a rectangle represents ROI of the tracked object + */ + CV_WRAP bool add(Ptr newTracker, InputArray image, const Rect2d& boundingBox); + + /** + * \brief Add a set of objects to be tracked. + * @param newTrackers list of tracking algorithms to be used + * @param image input image + * @param boundingBox list of the tracked objects + */ + bool add(std::vector > newTrackers, InputArray image, std::vector boundingBox); + + /** + * \brief Update the current tracking status. + * The result will be saved in the internal storage. + * @param image input image + */ + bool update(InputArray image); + + /** + * \brief Update the current tracking status. + * @param image input image + * @param boundingBox the tracking result, represent a list of ROIs of the tracked objects. + */ + CV_WRAP bool update(InputArray image, CV_OUT std::vector & boundingBox); + + /** + * \brief Returns a reference to a storage for the tracked objects, each object corresponds to one tracker algorithm + */ + CV_WRAP const std::vector& getObjects() const; + + /** + * \brief Returns a pointer to a new instance of MultiTracker + */ + CV_WRAP static Ptr create(); + +protected: + //!< storage for the tracker algorithms. + std::vector< Ptr > trackerList; + + //!< storage for the tracked objects, each object corresponds to one tracker algorithm. + std::vector objects; +}; + +/************************************ Multi-Tracker Classes ---By Tyan Vladimir---************************************/ + +/** @brief Base abstract class for the long-term Multi Object Trackers: + +@sa Tracker, MultiTrackerTLD +*/ +class CV_EXPORTS MultiTracker_Alt +{ +public: + /** @brief Constructor for Multitracker + */ + MultiTracker_Alt() + { + targetNum = 0; + } + + /** @brief Add a new target to a tracking-list and initialize the tracker with a known bounding box that surrounded the target + @param image The initial frame + @param boundingBox The initial bounding box of target + @param tracker_algorithm Multi-tracker algorithm + + @return True if new target initialization went succesfully, false otherwise + */ + bool addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm); + + /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets + @param image The current frame + + @return True means that all targets were located and false means that tracker couldn't locate one of the targets in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ + bool update(InputArray image); + + /** @brief Current number of targets in tracking-list + */ + int targetNum; + + /** @brief Trackers list for Multi-Object-Tracker + */ + std::vector > trackers; + + /** @brief Bounding Boxes list for Multi-Object-Tracker + */ + std::vector boundingBoxes; + /** @brief List of randomly generated colors for bounding boxes display + */ + std::vector colors; +}; + +/** @brief Multi Object %Tracker for TLD. + +TLD is a novel tracking framework that explicitly decomposes +the long-term tracking task into tracking, learning and detection. + +The tracker follows the object from frame to frame. The detector localizes all appearances that +have been observed so far and corrects the tracker if necessary. The learning estimates detector's +errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . + +The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this +implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial +occlusions, object absence etc. + +@sa Tracker, MultiTracker, TrackerTLD +*/ +class CV_EXPORTS MultiTrackerTLD : public MultiTracker_Alt +{ +public: + /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets by + optimized update method using some techniques to speedup calculations specifically for MO TLD. The only limitation + is that all target bounding boxes should have approximately same aspect ratios. Speed boost is around 20% + + @param image The current frame. + + @return True means that all targets were located and false means that tracker couldn't locate one of the targets in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ + bool update_opt(InputArray image); +}; + +/*********************************** CSRT ************************************/ +/** @brief the CSRT tracker + +The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability +*/ +class CV_EXPORTS_W TrackerCSRT : public cv::legacy::Tracker +{ +public: + struct CV_EXPORTS Params : cv::tracking::TrackerCSRT::Params + { + /** + * \brief Read parameters from a file + */ + void read(const FileNode& /*fn*/); + + /** + * \brief Write parameters to a file + */ + void write(cv::FileStorage& fs) const; + }; + + /** @brief Constructor + @param parameters CSRT parameters TrackerCSRT::Params + */ + static Ptr create(const TrackerCSRT::Params ¶meters); + + CV_WRAP static Ptr create(); + + CV_WRAP virtual void setInitialMask(InputArray mask) = 0; + + virtual ~TrackerCSRT() CV_OVERRIDE {} +}; + + +CV_EXPORTS_W Ptr upgradeTrackingAPI(const Ptr& legacy_tracker); + +//! @} + +#ifndef CV_DOXYGEN +} // namespace +#endif +}} // namespace + +#endif // OPENCV_TRACKING_LEGACY_HPP diff --git a/modules/tracking/misc/java/gen_dict.json b/modules/tracking/misc/java/gen_dict.json new file mode 100644 index 00000000000..a2c3bf0dacf --- /dev/null +++ b/modules/tracking/misc/java/gen_dict.json @@ -0,0 +1,5 @@ +{ + "namespaces_dict": { + "cv.legacy": "legacy" + } +} diff --git a/modules/tracking/misc/java/test/TrackerCreateLegacyTest.java b/modules/tracking/misc/java/test/TrackerCreateLegacyTest.java new file mode 100644 index 00000000000..3c8bfa9991e --- /dev/null +++ b/modules/tracking/misc/java/test/TrackerCreateLegacyTest.java @@ -0,0 +1,23 @@ +package org.opencv.test.tracking; + +import org.opencv.core.Core; +import org.opencv.core.CvException; +import org.opencv.test.OpenCVTestCase; + +import org.opencv.tracking.Tracking; +import org.opencv.tracking.legacy_Tracker; +import org.opencv.tracking.legacy_TrackerTLD; + +public class TrackerCreateLegacyTest extends OpenCVTestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + + public void testCreateLegacyTrackerTLD() { + legacy_Tracker tracker = legacy_TrackerTLD.create(); + } + +} diff --git a/modules/tracking/misc/java/test/TrackerCreateTest.java b/modules/tracking/misc/java/test/TrackerCreateTest.java new file mode 100644 index 00000000000..f4db6abadad --- /dev/null +++ b/modules/tracking/misc/java/test/TrackerCreateTest.java @@ -0,0 +1,37 @@ +package org.opencv.test.tracking; + +import org.opencv.core.Core; +import org.opencv.core.CvException; +import org.opencv.test.OpenCVTestCase; + +import org.opencv.video.Tracker; +import org.opencv.video.TrackerGOTURN; +import org.opencv.tracking.TrackerKCF; +import org.opencv.video.TrackerMIL; + +public class TrackerCreateTest extends OpenCVTestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + + public void testCreateTrackerGOTURN() { + try { + Tracker tracker = TrackerGOTURN.create(); + assert(tracker != null); + } catch (CvException e) { + // expected, model files may be missing + } + } + + public void testCreateTrackerKCF() { + Tracker tracker = TrackerKCF.create(); + } + + public void testCreateTrackerMIL() { + Tracker tracker = TrackerMIL.create(); + } + +} diff --git a/modules/tracking/misc/objc/gen_dict.json b/modules/tracking/misc/objc/gen_dict.json index 6cefc5f55ad..7172e2df474 100644 --- a/modules/tracking/misc/objc/gen_dict.json +++ b/modules/tracking/misc/objc/gen_dict.json @@ -4,5 +4,8 @@ }, "AdditionalImports" : { "*" : [ "\"tracking.hpp\"" ] - } + }, + "namespace_ignore_list" : [ + "cv.legacy" + ] } diff --git a/modules/tracking/misc/python/pyopencv_tracking.hpp b/modules/tracking/misc/python/pyopencv_tracking.hpp new file mode 100644 index 00000000000..10c4eeb9da2 --- /dev/null +++ b/modules/tracking/misc/python/pyopencv_tracking.hpp @@ -0,0 +1,4 @@ +#ifdef HAVE_OPENCV_TRACKING +typedef TrackerCSRT::Params TrackerCSRT_Params; +typedef TrackerKCF::Params TrackerKCF_Params; +#endif diff --git a/modules/tracking/misc/python/test/test_tracking_contrib.py b/modules/tracking/misc/python/test/test_tracking_contrib.py new file mode 100644 index 00000000000..7eeb91e1e36 --- /dev/null +++ b/modules/tracking/misc/python/test/test_tracking_contrib.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +import os +import numpy as np +import cv2 as cv + +from tests_common import NewOpenCVTests, unittest + +class tracking_contrib_test(NewOpenCVTests): + + def test_createTracker(self): + + t = cv.TrackerMIL_create() + t = cv.TrackerKCF_create() + try: + t = cv.TrackerGOTURN_create() + except cv.error as e: + pass # may fail due to missing DL model files + + def test_createLegacyTracker(self): + + t = cv.legacy.TrackerBoosting_create() + t = cv.legacy.TrackerMIL_create() + t = cv.legacy.TrackerKCF_create() + t = cv.legacy.TrackerMedianFlow_create() + #t = cv.legacy.TrackerGOTURN_create() + t = cv.legacy.TrackerMOSSE_create() + t = cv.legacy.TrackerCSRT_create() + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/tracking/perf/perf_Tracker.cpp b/modules/tracking/perf/perf_Tracker.cpp deleted file mode 100644 index 4667c713b25..00000000000 --- a/modules/tracking/perf/perf_Tracker.cpp +++ /dev/null @@ -1,412 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "perf_precomp.hpp" - -namespace opencv_test { namespace { - -//write sanity: ./bin/opencv_perf_tracking --perf_write_sanity=true --perf_min_samples=1 -//verify sanity: ./bin/opencv_perf_tracking --perf_min_samples=1 - -#define TESTSET_NAMES testing::Values("david","dudek","faceocc2") -//#define TESTSET_NAMES testing::internal::ValueArray1("david") -#define SEGMENTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - -const string TRACKING_DIR = "cv/tracking"; -const string FOLDER_IMG = "data"; - -typedef perf::TestBaseWithParam > tracking; - -std::vector splitString( std::string s, std::string delimiter ) -{ - std::vector token; - size_t pos = 0; - while ( ( pos = s.find( delimiter ) ) != std::string::npos ) - { - token.push_back( s.substr( 0, pos ) ); - s.erase( 0, pos + delimiter.length() ); - } - token.push_back( s ); - return token; -} - -void checkData( const string& datasetMeta, int& startFrame, string& prefix, string& suffix ) -{ - //get informations on the current test data - FileStorage fs; - fs.open( datasetMeta, FileStorage::READ ); - fs["start"] >> startFrame; - fs["prefix"] >> prefix; - fs["suffix"] >> suffix; - fs.release(); -} - -bool getGroundTruth( const string& gtFile, vector& gtBBs ) -{ - std::ifstream gt; - //open the ground truth - gt.open( gtFile.c_str() ); - if( !gt.is_open() ) - { - return false; - } - string line; - Rect currentBB; - while ( getline( gt, line ) ) - { - vector tokens = splitString( line, "," ); - - if( tokens.size() != 4 ) - { - return false; - } - - gtBBs.push_back( - Rect( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ) ); - } - return true; -} - -void getSegment( int segmentId, int numSegments, int bbCounter, int& startFrame, int& endFrame ) -{ - //compute the start and the and for each segment - int gtStartFrame = startFrame; - int numFrame = bbCounter / numSegments; - startFrame += ( segmentId - 1 ) * numFrame; - endFrame = startFrame + numFrame; - - if( ( segmentId ) == numSegments ) - endFrame = bbCounter + gtStartFrame - 1; -} - -void getMatOfRects( const vector& bbs, Mat& bbs_mat ) -{ - for ( int b = 0, size = (int)bbs.size(); b < size; b++ ) - { - bbs_mat.at( b, 0 ) = (float)bbs[b].x; - bbs_mat.at( b, 1 ) = (float)bbs[b].y; - bbs_mat.at( b, 2 ) = (float)bbs[b].width; - bbs_mat.at( b, 3 ) = (float)bbs[b].height; - } -} - -PERF_TEST_P(tracking, mil, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>( GetParam() ); - int segmentId = get<1>( GetParam() ); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); - checkData( datasetMeta, startFrame, prefix, suffix ); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); - if( !getGroundTruth( gtFile, gtBBs ) ) - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerMIL::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); - int endFrame = 0; - getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); - - for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) - { - c >> frame; - - if( frame.empty() ) - { - break; - } - - if( !initialized ) - { - if( !tracker->init( frame, currentBB ) ) - { - FAIL()<< "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if( initialized ) - { - tracker->update( frame, currentBB ); - } - bbs.push_back( currentBB ); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat( (int)bbs.size(), 4, CV_32F ); - getMatOfRects( bbs, bbs_mat ); - - SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); - -} - -PERF_TEST_P(tracking, boosting, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>( GetParam() ); - int segmentId = get<1>( GetParam() ); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); - checkData( datasetMeta, startFrame, prefix, suffix ); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); - if( !getGroundTruth( gtFile, gtBBs ) ) - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerBoosting::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); - int endFrame = 0; - getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); - for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) - { - c >> frame; - - if( frame.empty() ) - { - break; - } - - if( !initialized ) - { - if( !tracker->init( frame, currentBB ) ) - { - FAIL()<< "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if( initialized ) - { - tracker->update( frame, currentBB ); - } - bbs.push_back( currentBB ); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat( (int)bbs.size(), 4, CV_32F ); - getMatOfRects( bbs, bbs_mat ); - - SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); - -} - -PERF_TEST_P(tracking, tld, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>( GetParam() ); - int segmentId = get<1>( GetParam() ); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); - checkData( datasetMeta, startFrame, prefix, suffix ); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); - if( !getGroundTruth( gtFile, gtBBs ) ) - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerTLD::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); - int endFrame = 0; - getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); - for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) - { - c >> frame; - - if( frame.empty() ) - { - break; - } - - if( !initialized ) - { - if( !tracker->init( frame, currentBB ) ) - { - FAIL()<< "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if( initialized ) - { - tracker->update( frame, currentBB ); - } - bbs.push_back( currentBB ); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat( (int)bbs.size(), 4, CV_32F ); - getMatOfRects( bbs, bbs_mat ); - - SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); - -} - -PERF_TEST_P(tracking, GOTURN, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>(GetParam()); - int segmentId = get<1>(GetParam()); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath(TRACKING_DIR + "/" + video + "/" + video + ".yml"); - checkData(datasetMeta, startFrame, prefix, suffix); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath(TRACKING_DIR + "/" + video + "/gt.txt"); - if (!getGroundTruth(gtFile, gtBBs)) - FAIL() << "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerGOTURN::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = (sizeof(SEGMENTS) / sizeof(int)); - int endFrame = 0; - getSegment(segmentId, numSegments, bbCounter, startFrame, endFrame); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open(getDataPath(TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm")); - c.set(CAP_PROP_POS_FRAMES, startFrame); - for (int frameCounter = startFrame; frameCounter < endFrame; frameCounter++) - { - c >> frame; - - if (frame.empty()) - { - break; - } - - if (!initialized) - { - if (!tracker->init(frame, currentBB)) - { - FAIL() << "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if (initialized) - { - tracker->update(frame, currentBB); - } - bbs.push_back(currentBB); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat((int)bbs.size(), 4, CV_32F); - getMatOfRects(bbs, bbs_mat); - - SANITY_CHECK(bbs_mat, 15, ERROR_RELATIVE); - -} - -}} // namespace diff --git a/modules/tracking/perf/perf_main.cpp b/modules/tracking/perf/perf_main.cpp index cfd91a4dc7a..ccca051e660 100644 --- a/modules/tracking/perf/perf_main.cpp +++ b/modules/tracking/perf/perf_main.cpp @@ -3,4 +3,19 @@ // of this distribution and at http://opencv.org/license.html. #include "perf_precomp.hpp" -CV_PERF_TEST_MAIN(tracking) +static +void initTrackingTests() +{ + const char* extraTestDataPath = +#ifdef WINRT + NULL; +#else + getenv("OPENCV_DNN_TEST_DATA_PATH"); +#endif + if (extraTestDataPath) + cvtest::addDataSearchPath(extraTestDataPath); + + cvtest::addDataSearchSubDirectory(""); // override "cv" prefix below to access without "../dnn" hacks +} + +CV_PERF_TEST_MAIN(tracking, initTrackingTests()) diff --git a/modules/tracking/perf/perf_trackers.cpp b/modules/tracking/perf/perf_trackers.cpp new file mode 100644 index 00000000000..bd47b750fb0 --- /dev/null +++ b/modules/tracking/perf/perf_trackers.cpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "perf_precomp.hpp" + +#include + +namespace opencv_test { namespace { +using namespace perf; + +//using namespace cv::tracking; + +typedef tuple TrackingParams_t; + +std::vector getTrackingParams() +{ + std::vector params { + TrackingParams_t("david/data/david.webm", 300, Rect(163,62,47,56)), + TrackingParams_t("dudek/data/dudek.webm", 1, Rect(123,87,132,176)), + TrackingParams_t("faceocc2/data/faceocc2.webm", 1, Rect(118,57,82,98)) + }; + return params; +} + +class Tracking : public perf::TestBaseWithParam +{ +public: + template + void runTrackingTest(const Ptr& tracker, const TrackingParams_t& params); +}; + +template +void Tracking::runTrackingTest(const Ptr& tracker, const TrackingParams_t& params) +{ + const int N = 10; + string video = get<0>(params); + int startFrame = get<1>(params); + //int endFrame = startFrame + N; + Rect boundingBox = get<2>(params); + + string videoPath = findDataFile(std::string("cv/tracking/") + video); + + VideoCapture c; + c.open(videoPath); + ASSERT_TRUE(c.isOpened()) << videoPath; +#if 0 + // c.set(CAP_PROP_POS_FRAMES, startFrame); +#else + if (startFrame) + std::cout << "startFrame = " << startFrame << std::endl; + for (int i = 0; i < startFrame; i++) + { + Mat dummy_frame; + c >> dummy_frame; + ASSERT_FALSE(dummy_frame.empty()) << i << ": " << videoPath; + } +#endif + + // decode frames into memory (don't measure decoding performance) + std::vector frames; + for (int i = 0; i < N; ++i) + { + Mat frame; + c >> frame; + ASSERT_FALSE(frame.empty()) << "i=" << i; + frames.push_back(frame); + } + + std::cout << "frame size = " << frames[0].size() << std::endl; + + PERF_SAMPLE_BEGIN(); + { + tracker->init(frames[0], (ROI_t)boundingBox); + for (int i = 1; i < N; ++i) + { + ROI_t rc; + tracker->update(frames[i], rc); + ASSERT_FALSE(rc.empty()); + } + } + PERF_SAMPLE_END(); + + SANITY_CHECK_NOTHING(); +} + + +//================================================================================================== + +PERF_TEST_P(Tracking, Boosting, testing::ValuesIn(getTrackingParams())) +{ + auto tracker = legacy::TrackerBoosting::create(); + runTrackingTest(tracker, GetParam()); +} + +PERF_TEST_P(Tracking, TLD, testing::ValuesIn(getTrackingParams())) +{ + auto tracker = legacy::TrackerTLD::create(); + runTrackingTest(tracker, GetParam()); +} + +}} // namespace diff --git a/modules/tracking/samples/benchmark.cpp b/modules/tracking/samples/benchmark.cpp index 7becc6a03e5..935244f92b5 100644 --- a/modules/tracking/samples/benchmark.cpp +++ b/modules/tracking/samples/benchmark.cpp @@ -93,7 +93,7 @@ struct AlgoWrap Ptr tracker; bool lastRes; - Rect2d lastBox; + Rect lastBox; State lastState; // visual @@ -112,14 +112,14 @@ struct AlgoWrap void eval(const Mat &frame, const Rect2d >Box, bool isVerbose) { // RUN - lastBox = Rect2d(); + lastBox = Rect(); int64 frameTime = getTickCount(); lastRes = tracker->update(frame, lastBox); frameTime = getTickCount() - frameTime; // RESULTS - double intersectArea = (gtBox & lastBox).area(); - double unionArea = (gtBox | lastBox).area(); + double intersectArea = (gtBox & (Rect2d)lastBox).area(); + double unionArea = (gtBox | (Rect2d)lastBox).area(); numTotal++; numResponse += (lastRes && isGoodBox(lastBox)) ? 1 : 0; numPresent += isGoodBox(gtBox) ? 1 : 0; diff --git a/modules/tracking/samples/csrt.cpp b/modules/tracking/samples/csrt.cpp index 1f77e89a8c5..b52c8d37ac8 100644 --- a/modules/tracking/samples/csrt.cpp +++ b/modules/tracking/samples/csrt.cpp @@ -41,7 +41,7 @@ int main(int argc, char** argv) cap >> frame; // target bounding box - Rect2d roi; + Rect roi; if (argc > 2) { // read first line of ground-truth file std::string groundtruthPath = argv[2]; diff --git a/modules/tracking/samples/goturnTracker.cpp b/modules/tracking/samples/goturnTracker.cpp index a1e72171871..9bdaa97a129 100644 --- a/modules/tracking/samples/goturnTracker.cpp +++ b/modules/tracking/samples/goturnTracker.cpp @@ -50,6 +50,7 @@ #include "opencv2/datasets/track_alov.hpp" #include +#include #include #include #include @@ -65,7 +66,7 @@ static Mat image; static bool paused; static bool selectObjects = false; static bool startSelection = false; -Rect2d boundingBox; +static Rect boundingBox; static const char* keys = { "{@dataset_path || Dataset path }" @@ -140,7 +141,7 @@ int main(int argc, char *argv[]) setMouseCallback("GOTURN Tracking", onMouse, 0); //Create GOTURN tracker - Ptr tracker = TrackerGOTURN::create(); + auto tracker = TrackerGOTURN::create(); //Load and init full ALOV300++ dataset with a given datasetID, as alternative you can use loadAnnotatedOnly(..) //to load only frames with labelled ground truth ~ every 5-th frame @@ -181,11 +182,7 @@ int main(int argc, char *argv[]) if (!initialized && selectObjects) { //Initialize the tracker and add targets - if (!tracker->init(frame, boundingBox)) - { - cout << "Tracker Init Error!!!"; - return 0; - } + tracker->init(frame, boundingBox); rectangle(frame, boundingBox, Scalar(0, 0, 255), 2, 1); initialized = true; } diff --git a/modules/tracking/samples/kcf.cpp b/modules/tracking/samples/kcf.cpp index f40a3c5f266..e1d7ab20745 100644 --- a/modules/tracking/samples/kcf.cpp +++ b/modules/tracking/samples/kcf.cpp @@ -41,7 +41,7 @@ int main( int argc, char** argv ){ // get bounding box cap >> frame; - Rect2d roi= selectROI("tracker", frame, true, false); + Rect roi = selectROI("tracker", frame, true, false); //quit if ROI was not selected if(roi.width==0 || roi.height==0) diff --git a/modules/tracking/samples/multiTracker_dataset.cpp b/modules/tracking/samples/multiTracker_dataset.cpp index 622a69ac758..efaed5633c1 100644 --- a/modules/tracking/samples/multiTracker_dataset.cpp +++ b/modules/tracking/samples/multiTracker_dataset.cpp @@ -148,7 +148,7 @@ int main(int argc, char *argv[]) namedWindow("Tracking API", 0); setMouseCallback("Tracking API", onMouse, 0); - MultiTrackerTLD mt; + legacy::MultiTrackerTLD mt; //Init Dataset Ptr dataset = TRACK_vot::create(); dataset->load(datasetRootPath); @@ -185,7 +185,7 @@ int main(int argc, char *argv[]) //Initialize the tracker and add targets for (int i = 0; i < (int)boundingBoxes.size(); i++) { - if (!mt.addTarget(frame, boundingBoxes[i], createTrackerByName(tracker_algorithm))) + if (!mt.addTarget(frame, boundingBoxes[i], createTrackerByName_legacy(tracker_algorithm))) { cout << "Trackers Init Error!!!"; return 0; diff --git a/modules/tracking/samples/multitracker.cpp b/modules/tracking/samples/multitracker.cpp index 7b21334d503..62df0199192 100644 --- a/modules/tracking/samples/multitracker.cpp +++ b/modules/tracking/samples/multitracker.cpp @@ -73,7 +73,7 @@ int main( int argc, char** argv ){ trackingAlg = argv[2]; // create the tracker - MultiTracker trackers; + legacy::MultiTracker trackers; // container of the tracked objects vector ROIs; @@ -93,10 +93,10 @@ int main( int argc, char** argv ){ if(ROIs.size()<1) return 0; - std::vector > algorithms; + std::vector > algorithms; for (size_t i = 0; i < ROIs.size(); i++) { - algorithms.push_back(createTrackerByName(trackingAlg)); + algorithms.push_back(createTrackerByName_legacy(trackingAlg)); objects.push_back(ROIs[i]); } diff --git a/modules/tracking/samples/samples_utility.hpp b/modules/tracking/samples/samples_utility.hpp index de3525d507c..2fe876919ec 100644 --- a/modules/tracking/samples/samples_utility.hpp +++ b/modules/tracking/samples/samples_utility.hpp @@ -2,25 +2,28 @@ #define _SAMPLES_UTILITY_HPP_ #include +#include -inline cv::Ptr createTrackerByName(cv::String name) +inline cv::Ptr createTrackerByName(const std::string& name) { + using namespace cv; + cv::Ptr tracker; if (name == "KCF") tracker = cv::TrackerKCF::create(); else if (name == "TLD") - tracker = cv::TrackerTLD::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerTLD::create()); else if (name == "BOOSTING") - tracker = cv::TrackerBoosting::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerBoosting::create()); else if (name == "MEDIAN_FLOW") - tracker = cv::TrackerMedianFlow::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerMedianFlow::create()); else if (name == "MIL") tracker = cv::TrackerMIL::create(); else if (name == "GOTURN") tracker = cv::TrackerGOTURN::create(); else if (name == "MOSSE") - tracker = cv::TrackerMOSSE::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerMOSSE::create()); else if (name == "CSRT") tracker = cv::TrackerCSRT::create(); else @@ -29,4 +32,32 @@ inline cv::Ptr createTrackerByName(cv::String name) return tracker; } +inline cv::Ptr createTrackerByName_legacy(const std::string& name) +{ + using namespace cv; + + cv::Ptr tracker; + + if (name == "KCF") + tracker = legacy::TrackerKCF::create(); + else if (name == "TLD") + tracker = legacy::TrackerTLD::create(); + else if (name == "BOOSTING") + tracker = legacy::TrackerBoosting::create(); + else if (name == "MEDIAN_FLOW") + tracker = legacy::TrackerMedianFlow::create(); + else if (name == "MIL") + tracker = legacy::TrackerMIL::create(); + else if (name == "GOTURN") + CV_Error(cv::Error::StsNotImplemented, "FIXIT: migration on new API is required"); + else if (name == "MOSSE") + tracker = legacy::TrackerMOSSE::create(); + else if (name == "CSRT") + tracker = legacy::TrackerCSRT::create(); + else + CV_Error(cv::Error::StsBadArg, "Invalid tracking algorithm name\n"); + + return tracker; +} + #endif diff --git a/modules/tracking/samples/tracker.cpp b/modules/tracking/samples/tracker.cpp index add014c8a1c..5de5e60f616 100644 --- a/modules/tracking/samples/tracker.cpp +++ b/modules/tracking/samples/tracker.cpp @@ -88,7 +88,7 @@ int main( int argc, char** argv ){ namedWindow( "Tracking API", 1 ); Mat image; - Rect2d boundingBox; + Rect boundingBox; bool paused = false; //instantiates the specific Tracker @@ -134,11 +134,7 @@ int main( int argc, char** argv ){ if( !initialized ) { //initializes the tracker - if( !tracker->init( frame, boundingBox ) ) - { - cout << "***Could not initialize tracker...***\n"; - return -1; - } + tracker->init(frame, boundingBox); initialized = true; } else if( initialized ) diff --git a/modules/tracking/samples/tracker_dataset.cpp b/modules/tracking/samples/tracker_dataset.cpp index 1dbeb4c8407..479546bb77d 100644 --- a/modules/tracking/samples/tracker_dataset.cpp +++ b/modules/tracking/samples/tracker_dataset.cpp @@ -66,7 +66,7 @@ using namespace cv::datasets; //#define RECORD_VIDEO_FLG static Mat image; -static Rect2d boundingBox; +static Rect boundingBox; static bool paused; static bool selectObject = false; static bool startSelection = false; @@ -186,11 +186,7 @@ int main(int argc, char *argv[]) if (!initialized && selectObject) { //initializes the tracker - if (!tracker->init(frame, boundingBox)) - { - cout << "***Could not initialize tracker...***\n"; - return -1; - } + tracker->init(frame, boundingBox); initialized = true; } else if (initialized) diff --git a/modules/tracking/samples/tracking_by_matching.cpp b/modules/tracking/samples/tracking_by_matching.cpp index 1e53845a1e5..d2e10afd683 100644 --- a/modules/tracking/samples/tracking_by_matching.cpp +++ b/modules/tracking/samples/tracking_by_matching.cpp @@ -8,7 +8,8 @@ using namespace std; using namespace cv; -using namespace cv::tbm; +using namespace cv::detail::tracking; +using namespace cv::detail::tracking::tbm; static const char* keys = { "{video_name | | video name }" @@ -123,7 +124,7 @@ class DnnObjectDetector cv::Ptr createTrackerByMatchingWithFastDescriptor() { - cv::tbm::TrackerParams params; + tbm::TrackerParams params; cv::Ptr tracker = createTrackerByMatching(params); diff --git a/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp b/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp index cd4418b2d2a..58e2beb03dd 100644 --- a/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp +++ b/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp @@ -25,7 +25,7 @@ int main( int argc, char** argv ){ } // declares all required variables - Rect2d roi; + Rect roi; Mat frame; //! [param] diff --git a/modules/tracking/samples/tutorial_introduction_to_tracker.cpp b/modules/tracking/samples/tutorial_introduction_to_tracker.cpp index e092c7d9946..4083c32d3df 100644 --- a/modules/tracking/samples/tutorial_introduction_to_tracker.cpp +++ b/modules/tracking/samples/tutorial_introduction_to_tracker.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -24,7 +25,7 @@ int main( int argc, char** argv ){ // declares all required variables //! [vars] - Rect2d roi; + Rect roi; Mat frame; //! [vars] diff --git a/modules/tracking/samples/tutorial_multitracker.cpp b/modules/tracking/samples/tutorial_multitracker.cpp index 0c6d218006b..efb9af881f6 100644 --- a/modules/tracking/samples/tutorial_multitracker.cpp +++ b/modules/tracking/samples/tutorial_multitracker.cpp @@ -40,7 +40,7 @@ int main( int argc, char** argv ){ // create the tracker //! [create] - MultiTracker trackers; + legacy::MultiTracker trackers; //! [create] // container of the tracked objects @@ -67,10 +67,10 @@ int main( int argc, char** argv ){ // initialize the tracker //! [init] - std::vector > algorithms; + std::vector > algorithms; for (size_t i = 0; i < ROIs.size(); i++) { - algorithms.push_back(createTrackerByName(trackingAlg)); + algorithms.push_back(createTrackerByName_legacy(trackingAlg)); objects.push_back(ROIs[i]); } diff --git a/modules/tracking/src/PFSolver.hpp b/modules/tracking/src/PFSolver.hpp index 520fa5f7e51..55c5b8193cb 100644 --- a/modules/tracking/src/PFSolver.hpp +++ b/modules/tracking/src/PFSolver.hpp @@ -1,5 +1,3 @@ -#include "opencv2/core.hpp" -#include "opencv2/core/core_c.h" #include #include #include diff --git a/modules/tracking/src/augmented_unscented_kalman.cpp b/modules/tracking/src/augmented_unscented_kalman.cpp index 85fdfabc42e..efc060b026b 100644 --- a/modules/tracking/src/augmented_unscented_kalman.cpp +++ b/modules/tracking/src/augmented_unscented_kalman.cpp @@ -42,10 +42,10 @@ #include "precomp.hpp" #include "opencv2/tracking/kalman_filters.hpp" -namespace cv -{ -namespace tracking -{ +namespace cv { +namespace detail { +inline namespace tracking { +inline namespace kalman_filters { void AugmentedUnscentedKalmanFilterParams:: init( int dp, int mp, int cp, double processNoiseCovDiag, double measurementNoiseCovDiag, @@ -394,5 +394,4 @@ Ptr createAugmentedUnscentedKalmanFilter(const AugmentedU return kfu; } -} // tracking -} // cv +}}}} // namespace diff --git a/modules/tracking/src/feature.cpp b/modules/tracking/src/feature.cpp index 7b8bfcff731..ecbceb57241 100644 --- a/modules/tracking/src/feature.cpp +++ b/modules/tracking/src/feature.cpp @@ -42,8 +42,9 @@ #include "precomp.hpp" #include "opencv2/tracking/feature.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TODO This implementation is based on apps/traincascade/ @@ -309,14 +310,14 @@ void CvHaarEvaluator::FeatureHaar::generateRandomFeature( Size patchSize ) position.y = rand() % ( patchSize.height ); position.x = rand() % ( patchSize.width ); - baseDim.width = (int) ( ( 1 - sqrt( 1 - (float) rand() / RAND_MAX ) ) * patchSize.width ); - baseDim.height = (int) ( ( 1 - sqrt( 1 - (float) rand() / RAND_MAX ) ) * patchSize.height ); + baseDim.width = (int) ( ( 1 - sqrt( 1 - (float) rand() * (float)(1.0 / RAND_MAX) ) ) * patchSize.width ); + baseDim.height = (int) ( ( 1 - sqrt( 1 - (float) rand() * (float)(1.0 / RAND_MAX) ) ) * patchSize.height ); //select types //float probType[11] = {0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0950f}; float probType[11] = { 0.2f, 0.2f, 0.2f, 0.2f, 0.2f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; - float prob = (float) rand() / RAND_MAX; + float prob = (float) rand() * (float)(1.0 / RAND_MAX); if( prob < probType[0] ) { @@ -686,7 +687,7 @@ void CvHaarEvaluator::FeatureHaar::generateRandomFeature( Size patchSize ) valid = true; } else - CV_Error(CV_StsAssert, ""); + CV_Error(Error::StsAssert, ""); } m_initSize = patchSize; @@ -1069,4 +1070,4 @@ void CvLBPEvaluator::Feature::write( FileStorage &fs ) const fs << CC_RECT << "[:" << rect.x << rect.y << rect.width << rect.height << "]"; } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/featureColorName.cpp b/modules/tracking/src/featureColorName.cpp index cb3aa3cf477..48bc360a50f 100644 --- a/modules/tracking/src/featureColorName.cpp +++ b/modules/tracking/src/featureColorName.cpp @@ -42,7 +42,10 @@ #include "precomp.hpp" #include -namespace cv{ +namespace cv { +namespace detail { +inline namespace tracking { + const float ColorNames[][10]={ {0.45975f,0.014802f,0.044289f,-0.028193f,0.001151f,-0.0050145f,0.34522f,0.018362f,0.23994f,0.1689f}, {0.47157f,0.021424f,0.041444f,-0.030215f,0.0019002f,-0.0029264f,0.32875f,0.0082059f,0.2502f,0.17007f}, @@ -32813,4 +32816,5 @@ namespace cv{ {0.0030858f,-0.016151f,0.013017f,0.0072284f,-0.53357f,0.30985f,0.0041336f,-0.012531f,0.00142f,-0.33842f}, {0.0087778f,-0.015645f,0.004769f,0.011785f,-0.54199f,0.31505f,0.00020476f,-0.020282f,0.00021236f,-0.34675f} }; -} + +}}} // namespace diff --git a/modules/tracking/src/gtrTracker.cpp b/modules/tracking/src/gtrTracker.cpp deleted file mode 100644 index c56bb5bbaff..00000000000 --- a/modules/tracking/src/gtrTracker.cpp +++ /dev/null @@ -1,189 +0,0 @@ -/*/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ -#include "opencv2/opencv_modules.hpp" -#include "gtrTracker.hpp" - - -namespace cv -{ - -TrackerGOTURN::Params::Params() -{ - modelTxt = "goturn.prototxt"; - modelBin = "goturn.caffemodel"; -} - -void TrackerGOTURN::Params::read(const cv::FileNode& /*fn*/){} - -void TrackerGOTURN::Params::write(cv::FileStorage& /*fs*/) const {} - - -Ptr TrackerGOTURN::create(const TrackerGOTURN::Params ¶meters) -{ -#ifdef HAVE_OPENCV_DNN - return Ptr(new gtr::TrackerGOTURNImpl(parameters)); -#else - (void)(parameters); - CV_Error(cv::Error::StsNotImplemented , "to use GOTURN, the tracking module needs to be built with opencv_dnn !"); -#endif -} -Ptr TrackerGOTURN::create() -{ - return TrackerGOTURN::create(TrackerGOTURN::Params()); -} - - -#ifdef HAVE_OPENCV_DNN -namespace gtr -{ - -class TrackerGOTURNModel : public TrackerModel{ -public: - TrackerGOTURNModel(TrackerGOTURN::Params){} - Rect2d getBoundingBox(){ return boundingBox_; } - void setBoudingBox(Rect2d boundingBox) { - if (image_.empty()) - CV_Error(Error::StsInternal, "Set image first"); - boundingBox_ = boundingBox & Rect2d(Point(0, 0), image_.size()); - } - Mat getImage(){ return image_; } - void setImage(const Mat& image){ image.copyTo(image_); } -protected: - Rect2d boundingBox_; - Mat image_; - void modelEstimationImpl(const std::vector&) CV_OVERRIDE {} - void modelUpdateImpl() CV_OVERRIDE {} -}; - -TrackerGOTURNImpl::TrackerGOTURNImpl(const TrackerGOTURN::Params ¶meters) : - params(parameters){ - isInit = false; -}; - -void TrackerGOTURNImpl::read(const cv::FileNode& fn) -{ - params.read(fn); -} - -void TrackerGOTURNImpl::write(cv::FileStorage& fs) const -{ - params.write(fs); -} - -bool TrackerGOTURNImpl::initImpl(const Mat& image, const Rect2d& boundingBox) -{ - //Make a simple model from frame and bounding box - model = Ptr(new TrackerGOTURNModel(params)); - ((TrackerGOTURNModel*)static_cast(model))->setImage(image); - ((TrackerGOTURNModel*)static_cast(model))->setBoudingBox(boundingBox); - - //Load GOTURN architecture from *.prototxt and pretrained weights from *.caffemodel - net = dnn::readNetFromCaffe(params.modelTxt, params.modelBin); - return true; -} - -bool TrackerGOTURNImpl::updateImpl(const Mat& image, Rect2d& boundingBox) -{ - int INPUT_SIZE = 227; - //Using prevFrame & prevBB from model and curFrame GOTURN calculating curBB - Mat curFrame = image.clone(); - Mat prevFrame = ((TrackerGOTURNModel*)static_cast(model))->getImage(); - Rect2d prevBB = ((TrackerGOTURNModel*)static_cast(model))->getBoundingBox(); - Rect2d curBB; - - float padTargetPatch = 2.0; - Rect2f searchPatchRect, targetPatchRect; - Point2f currCenter, prevCenter; - Mat prevFramePadded, curFramePadded; - Mat searchPatch, targetPatch; - - prevCenter.x = (float)(prevBB.x + prevBB.width / 2); - prevCenter.y = (float)(prevBB.y + prevBB.height / 2); - - targetPatchRect.width = (float)(prevBB.width*padTargetPatch); - targetPatchRect.height = (float)(prevBB.height*padTargetPatch); - targetPatchRect.x = (float)(prevCenter.x - prevBB.width*padTargetPatch / 2.0 + targetPatchRect.width); - targetPatchRect.y = (float)(prevCenter.y - prevBB.height*padTargetPatch / 2.0 + targetPatchRect.height); - - targetPatchRect.width = std::min(targetPatchRect.width, (float)prevFrame.cols); - targetPatchRect.height = std::min(targetPatchRect.height, (float)prevFrame.rows); - targetPatchRect.x = std::max(-prevFrame.cols * 0.5f, std::min(targetPatchRect.x, prevFrame.cols * 1.5f)); - targetPatchRect.y = std::max(-prevFrame.rows * 0.5f, std::min(targetPatchRect.y, prevFrame.rows * 1.5f)); - - copyMakeBorder(prevFrame, prevFramePadded, (int)targetPatchRect.height, (int)targetPatchRect.height, (int)targetPatchRect.width, (int)targetPatchRect.width, BORDER_REPLICATE); - targetPatch = prevFramePadded(targetPatchRect).clone(); - - copyMakeBorder(curFrame, curFramePadded, (int)targetPatchRect.height, (int)targetPatchRect.height, (int)targetPatchRect.width, (int)targetPatchRect.width, BORDER_REPLICATE); - searchPatch = curFramePadded(targetPatchRect).clone(); - - //Preprocess - //Resize - resize(targetPatch, targetPatch, Size(INPUT_SIZE, INPUT_SIZE), 0, 0, INTER_LINEAR_EXACT); - resize(searchPatch, searchPatch, Size(INPUT_SIZE, INPUT_SIZE), 0, 0, INTER_LINEAR_EXACT); - - //Convert to Float type and subtract mean - Mat targetBlob = dnn::blobFromImage(targetPatch, 1.0f, Size(), Scalar::all(128), false); - Mat searchBlob = dnn::blobFromImage(searchPatch, 1.0f, Size(), Scalar::all(128), false); - - net.setInput(targetBlob, "data1"); - net.setInput(searchBlob, "data2"); - - Mat resMat = net.forward("scale").reshape(1, 1); - - curBB.x = targetPatchRect.x + (resMat.at(0) * targetPatchRect.width / INPUT_SIZE) - targetPatchRect.width; - curBB.y = targetPatchRect.y + (resMat.at(1) * targetPatchRect.height / INPUT_SIZE) - targetPatchRect.height; - curBB.width = (resMat.at(2) - resMat.at(0)) * targetPatchRect.width / INPUT_SIZE; - curBB.height = (resMat.at(3) - resMat.at(1)) * targetPatchRect.height / INPUT_SIZE; - - //Predicted BB - boundingBox = curBB; - - //Set new model image and BB from current frame - ((TrackerGOTURNModel*)static_cast(model))->setImage(curFrame); - ((TrackerGOTURNModel*)static_cast(model))->setBoudingBox(curBB); - - return true; -} - -} -#endif // OPENCV_HAVE_DNN - -} diff --git a/modules/tracking/src/gtrTracker.hpp b/modules/tracking/src/gtrTracker.hpp deleted file mode 100644 index cd2820da5fd..00000000000 --- a/modules/tracking/src/gtrTracker.hpp +++ /dev/null @@ -1,80 +0,0 @@ - -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#ifndef OPENCV_GOTURN_TRACKER -#define OPENCV_GOTURN_TRACKER - -#include "precomp.hpp" -#include "opencv2/video/tracking.hpp" -#include "gtrUtils.hpp" -#include "opencv2/imgproc.hpp" - -#include -#include - -#include "opencv2/opencv_modules.hpp" -#ifdef HAVE_OPENCV_DNN -#include "opencv2/dnn.hpp" - -namespace cv -{ -namespace gtr -{ - -class TrackerGOTURNImpl : public TrackerGOTURN -{ -public: - TrackerGOTURNImpl(const TrackerGOTURN::Params ¶meters = TrackerGOTURN::Params()); - void read(const FileNode& fn) CV_OVERRIDE; - void write(FileStorage& fs) const CV_OVERRIDE; - bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE; - bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE; - - TrackerGOTURN::Params params; - - dnn::Net net; -}; - -} -} -#endif -#endif diff --git a/modules/tracking/src/gtrUtils.cpp b/modules/tracking/src/gtrUtils.cpp index e80dda1eadf..d2be7588d37 100644 --- a/modules/tracking/src/gtrUtils.cpp +++ b/modules/tracking/src/gtrUtils.cpp @@ -39,6 +39,7 @@ // //M*/ +#include "precomp.hpp" #include "gtrUtils.hpp" diff --git a/modules/tracking/src/gtrUtils.hpp b/modules/tracking/src/gtrUtils.hpp index fedf26f8864..0a3dea5adab 100644 --- a/modules/tracking/src/gtrUtils.hpp +++ b/modules/tracking/src/gtrUtils.hpp @@ -1,7 +1,6 @@ #ifndef OPENCV_GTR_UTILS #define OPENCV_GTR_UTILS -#include "precomp.hpp" #include namespace cv diff --git a/modules/tracking/src/kuhn_munkres.cpp b/modules/tracking/src/kuhn_munkres.cpp index 1d33c70e42c..5d0acee85d5 100644 --- a/modules/tracking/src/kuhn_munkres.cpp +++ b/modules/tracking/src/kuhn_munkres.cpp @@ -2,12 +2,17 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +#include "precomp.hpp" #include "kuhn_munkres.hpp" #include #include #include +namespace cv { +namespace detail { +inline namespace tracking { + KuhnMunkres::KuhnMunkres() : n_() {} std::vector KuhnMunkres::Solve(const cv::Mat& dissimilarity_matrix) { @@ -166,3 +171,6 @@ void KuhnMunkres::Run() { } } } + + +}}} // namespace \ No newline at end of file diff --git a/modules/tracking/src/kuhn_munkres.hpp b/modules/tracking/src/kuhn_munkres.hpp index 4f5ea28c185..d093d33af27 100644 --- a/modules/tracking/src/kuhn_munkres.hpp +++ b/modules/tracking/src/kuhn_munkres.hpp @@ -10,6 +10,9 @@ #include #include +namespace cv { +namespace detail { +inline namespace tracking { /// /// \brief The KuhnMunkres class @@ -52,4 +55,6 @@ class KuhnMunkres { int FindInCol(int col, int what); void Run(); }; + +}}} // namespace #endif // #ifndef __OPENCV_TRACKING_KUHN_MUNKRES_HPP__ diff --git a/modules/tracking/include/opencv2/tracking/onlineMIL.hpp b/modules/tracking/src/legacy/tracker.legacy.hpp similarity index 51% rename from modules/tracking/include/opencv2/tracking/onlineMIL.hpp rename to modules/tracking/src/legacy/tracker.legacy.hpp index 78e1372f8a3..2622ffbba32 100644 --- a/modules/tracking/include/opencv2/tracking/onlineMIL.hpp +++ b/modules/tracking/src/legacy/tracker.legacy.hpp @@ -39,80 +39,96 @@ // //M*/ -#ifndef __OPENCV_ONLINEMIL_HPP__ -#define __OPENCV_ONLINEMIL_HPP__ +#include "opencv2/tracking/tracking_legacy.hpp" -#include "opencv2/core.hpp" -#include +namespace cv { +namespace legacy { +inline namespace tracking { -namespace cv +Tracker::Tracker() { + isInit = false; +} -//! @addtogroup tracking -//! @{ +Tracker::~Tracker() +{ +} -//TODO based on the original implementation -//http://vision.ucsd.edu/~bbabenko/project_miltrack.shtml +bool Tracker::init( InputArray image, const Rect2d& boundingBox ) +{ -class ClfOnlineStump; + if( isInit ) + { + return false; + } -class CV_EXPORTS ClfMilBoost -{ - public: - struct CV_EXPORTS Params + if( image.empty() ) + return false; + + sampler = Ptr( new TrackerContribSampler() ); + featureSet = Ptr( new TrackerContribFeatureSet() ); + model = Ptr(); + + bool initTracker = initImpl( image.getMat(), boundingBox ); + + if (initTracker) { - Params(); - int _numSel; - int _numFeat; - float _lRate; - }; - - ClfMilBoost(); - ~ClfMilBoost(); - void init( const ClfMilBoost::Params ¶meters = ClfMilBoost::Params() ); - void update( const Mat& posx, const Mat& negx ); - std::vector classify( const Mat& x, bool logR = true ); - - inline float sigmoid( float x ) + isInit = true; + } + + return initTracker; +} + +bool Tracker::update( InputArray image, Rect2d& boundingBox ) +{ + + if( !isInit ) { - return 1.0f / ( 1.0f + exp( -x ) ); + return false; } - private: - uint _numsamples; - ClfMilBoost::Params _myParams; - std::vector _selectors; - std::vector _weakclf; - uint _counter; + if( image.empty() ) + return false; + + return updateImpl( image.getMat(), boundingBox ); +} -}; -class ClfOnlineStump -{ - public: - float _mu0, _mu1, _sig0, _sig1; - float _q; - int _s; - float _log_n1, _log_n0; - float _e1, _e0; - float _lRate; - - ClfOnlineStump(); - ClfOnlineStump( int ind ); - void init(); - void update( const Mat& posx, const Mat& negx, const cv::Mat_ & posw = cv::Mat_(), const cv::Mat_ & negw = cv::Mat_() ); - bool classify( const Mat& x, int i ); - float classifyF( const Mat& x, int i ); - std::vector classifySetF( const Mat& x ); - - private: - bool _trained; - int _ind; +class LegacyTrackerWrapper : public cv::Tracker +{ + const Ptr legacy_tracker_; +public: + LegacyTrackerWrapper(const Ptr& legacy_tracker) : legacy_tracker_(legacy_tracker) + { + CV_Assert(legacy_tracker_); + } + virtual ~LegacyTrackerWrapper() CV_OVERRIDE {}; + + void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE + { + CV_DbgAssert(legacy_tracker_); + legacy_tracker_->init(image, (Rect2d)boundingBox); + } + + bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE + { + CV_DbgAssert(legacy_tracker_); + Rect2d boundingBox2d; + bool res = legacy_tracker_->update(image, boundingBox2d); + int x1 = cvRound(boundingBox2d.x); + int y1 = cvRound(boundingBox2d.y); + int x2 = cvRound(boundingBox2d.x + boundingBox2d.width); + int y2 = cvRound(boundingBox2d.y + boundingBox2d.height); + boundingBox = Rect(x1, y1, x2 - x1, y2 - y1) & Rect(Point(0, 0), image.size()); + return res; + } }; -//! @} -} /* namespace cv */ +CV_EXPORTS_W Ptr upgradeTrackingAPI(const Ptr& legacy_tracker) +{ + return makePtr(legacy_tracker); +} -#endif +}}} // namespace diff --git a/modules/tracking/src/legacy/trackerCSRT.legacy.hpp b/modules/tracking/src/legacy/trackerCSRT.legacy.hpp new file mode 100644 index 00000000000..c42d270bd8b --- /dev/null +++ b/modules/tracking/src/legacy/trackerCSRT.legacy.hpp @@ -0,0 +1,159 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +namespace legacy { +inline namespace tracking { +namespace impl { + +class TrackerCSRTImpl CV_FINAL : public legacy::TrackerCSRT +{ +public: + cv::tracking::impl::TrackerCSRTImpl impl; + + TrackerCSRTImpl(const legacy::TrackerCSRT::Params ¶meters) + : impl(parameters) + { + isInit = false; + } + + void read(const FileNode& fn) CV_OVERRIDE + { + static_cast(impl.params).read(fn); + } + void write(FileStorage& fs) const CV_OVERRIDE + { + static_cast(impl.params).write(fs); + } + + bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE + { + impl.init(image, boundingBox); + model = impl.model; + sampler = makePtr(); + featureSet = makePtr(); + isInit = true; + return true; + } + bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE + { + Rect bb; + bool res = impl.update(image, bb); + boundingBox = bb; + return res; + } + + virtual void setInitialMask(InputArray mask) CV_OVERRIDE + { + impl.setInitialMask(mask); + } +}; + +} // namespace + +void legacy::TrackerCSRT::Params::read(const FileNode& fn) +{ + *this = TrackerCSRT::Params(); + if(!fn["padding"].empty()) + fn["padding"] >> padding; + if(!fn["template_size"].empty()) + fn["template_size"] >> template_size; + if(!fn["gsl_sigma"].empty()) + fn["gsl_sigma"] >> gsl_sigma; + if(!fn["hog_orientations"].empty()) + fn["hog_orientations"] >> hog_orientations; + if(!fn["num_hog_channels_used"].empty()) + fn["num_hog_channels_used"] >> num_hog_channels_used; + if(!fn["hog_clip"].empty()) + fn["hog_clip"] >> hog_clip; + if(!fn["use_hog"].empty()) + fn["use_hog"] >> use_hog; + if(!fn["use_color_names"].empty()) + fn["use_color_names"] >> use_color_names; + if(!fn["use_gray"].empty()) + fn["use_gray"] >> use_gray; + if(!fn["use_rgb"].empty()) + fn["use_rgb"] >> use_rgb; + if(!fn["window_function"].empty()) + fn["window_function"] >> window_function; + if(!fn["kaiser_alpha"].empty()) + fn["kaiser_alpha"] >> kaiser_alpha; + if(!fn["cheb_attenuation"].empty()) + fn["cheb_attenuation"] >> cheb_attenuation; + if(!fn["filter_lr"].empty()) + fn["filter_lr"] >> filter_lr; + if(!fn["admm_iterations"].empty()) + fn["admm_iterations"] >> admm_iterations; + if(!fn["number_of_scales"].empty()) + fn["number_of_scales"] >> number_of_scales; + if(!fn["scale_sigma_factor"].empty()) + fn["scale_sigma_factor"] >> scale_sigma_factor; + if(!fn["scale_model_max_area"].empty()) + fn["scale_model_max_area"] >> scale_model_max_area; + if(!fn["scale_lr"].empty()) + fn["scale_lr"] >> scale_lr; + if(!fn["scale_step"].empty()) + fn["scale_step"] >> scale_step; + if(!fn["use_channel_weights"].empty()) + fn["use_channel_weights"] >> use_channel_weights; + if(!fn["weights_lr"].empty()) + fn["weights_lr"] >> weights_lr; + if(!fn["use_segmentation"].empty()) + fn["use_segmentation"] >> use_segmentation; + if(!fn["histogram_bins"].empty()) + fn["histogram_bins"] >> histogram_bins; + if(!fn["background_ratio"].empty()) + fn["background_ratio"] >> background_ratio; + if(!fn["histogram_lr"].empty()) + fn["histogram_lr"] >> histogram_lr; + if(!fn["psr_threshold"].empty()) + fn["psr_threshold"] >> psr_threshold; + CV_Assert(number_of_scales % 2 == 1); + CV_Assert(use_gray || use_color_names || use_hog || use_rgb); +} +void legacy::TrackerCSRT::Params::write(FileStorage& fs) const +{ + fs << "padding" << padding; + fs << "template_size" << template_size; + fs << "gsl_sigma" << gsl_sigma; + fs << "hog_orientations" << hog_orientations; + fs << "num_hog_channels_used" << num_hog_channels_used; + fs << "hog_clip" << hog_clip; + fs << "use_hog" << use_hog; + fs << "use_color_names" << use_color_names; + fs << "use_gray" << use_gray; + fs << "use_rgb" << use_rgb; + fs << "window_function" << window_function; + fs << "kaiser_alpha" << kaiser_alpha; + fs << "cheb_attenuation" << cheb_attenuation; + fs << "filter_lr" << filter_lr; + fs << "admm_iterations" << admm_iterations; + fs << "number_of_scales" << number_of_scales; + fs << "scale_sigma_factor" << scale_sigma_factor; + fs << "scale_model_max_area" << scale_model_max_area; + fs << "scale_lr" << scale_lr; + fs << "scale_step" << scale_step; + fs << "use_channel_weights" << use_channel_weights; + fs << "weights_lr" << weights_lr; + fs << "use_segmentation" << use_segmentation; + fs << "histogram_bins" << histogram_bins; + fs << "background_ratio" << background_ratio; + fs << "histogram_lr" << histogram_lr; + fs << "psr_threshold" << psr_threshold; +} + +}} // namespace + +Ptr legacy::TrackerCSRT::create(const legacy::TrackerCSRT::Params ¶meters) +{ + return makePtr(parameters); +} +Ptr legacy::TrackerCSRT::create() +{ + return create(legacy::TrackerCSRT::Params()); +} + +} // namespace diff --git a/modules/tracking/src/legacy/trackerKCF.legacy.hpp b/modules/tracking/src/legacy/trackerKCF.legacy.hpp new file mode 100644 index 00000000000..e10284411e9 --- /dev/null +++ b/modules/tracking/src/legacy/trackerKCF.legacy.hpp @@ -0,0 +1,173 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +namespace legacy { +inline namespace tracking { +namespace impl { + +/*--------------------------- +| TrackerKCF +|---------------------------*/ +class TrackerKCFImpl CV_FINAL : public legacy::TrackerKCF +{ +public: + cv::tracking::impl::TrackerKCFImpl impl; + + TrackerKCFImpl(const legacy::TrackerKCF::Params ¶meters) + : impl(parameters) + { + isInit = false; + } + void read(const FileNode& fn) CV_OVERRIDE + { + static_cast(impl.params).read(fn); + } + void write(FileStorage& fs) const CV_OVERRIDE + { + static_cast(impl.params).write(fs); + } + + bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE + { + impl.init(image, boundingBox); + model = impl.model; + sampler = makePtr(); + featureSet = makePtr(); + isInit = true; + return true; + } + bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE + { + Rect bb; + bool res = impl.update(image, bb); + boundingBox = bb; + return res; + } + void setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func = false) CV_OVERRIDE + { + impl.setFeatureExtractor(f, pca_func); + } +}; + +} // namespace + +void legacy::TrackerKCF::Params::read(const cv::FileNode& fn) +{ + *this = TrackerKCF::Params(); + + if (!fn["detect_thresh"].empty()) + fn["detect_thresh"] >> detect_thresh; + + if (!fn["sigma"].empty()) + fn["sigma"] >> sigma; + + if (!fn["lambda"].empty()) + fn["lambda"] >> lambda; + + if (!fn["interp_factor"].empty()) + fn["interp_factor"] >> interp_factor; + + if (!fn["output_sigma_factor"].empty()) + fn["output_sigma_factor"] >> output_sigma_factor; + + if (!fn["resize"].empty()) + fn["resize"] >> resize; + + if (!fn["max_patch_size"].empty()) + fn["max_patch_size"] >> max_patch_size; + + if (!fn["split_coeff"].empty()) + fn["split_coeff"] >> split_coeff; + + if (!fn["wrap_kernel"].empty()) + fn["wrap_kernel"] >> wrap_kernel; + + + if (!fn["desc_npca"].empty()) + fn["desc_npca"] >> desc_npca; + + if (!fn["desc_pca"].empty()) + fn["desc_pca"] >> desc_pca; + + if (!fn["compress_feature"].empty()) + fn["compress_feature"] >> compress_feature; + + if (!fn["compressed_size"].empty()) + fn["compressed_size"] >> compressed_size; + + if (!fn["pca_learning_rate"].empty()) + fn["pca_learning_rate"] >> pca_learning_rate; +} + +void legacy::TrackerKCF::Params::write(cv::FileStorage& fs) const +{ + fs << "detect_thresh" << detect_thresh; + fs << "sigma" << sigma; + fs << "lambda" << lambda; + fs << "interp_factor" << interp_factor; + fs << "output_sigma_factor" << output_sigma_factor; + fs << "resize" << resize; + fs << "max_patch_size" << max_patch_size; + fs << "split_coeff" << split_coeff; + fs << "wrap_kernel" << wrap_kernel; + fs << "desc_npca" << desc_npca; + fs << "desc_pca" << desc_pca; + fs << "compress_feature" << compress_feature; + fs << "compressed_size" << compressed_size; + fs << "pca_learning_rate" << pca_learning_rate; +} + + +}} // namespace legacy::tracking + +Ptr legacy::TrackerKCF::create(const legacy::TrackerKCF::Params ¶meters) +{ + return makePtr(parameters); +} +Ptr legacy::TrackerKCF::create() +{ + return create(legacy::TrackerKCF::Params()); +} + +} diff --git a/modules/tracking/src/mosseTracker.cpp b/modules/tracking/src/mosseTracker.cpp index 038a8ba1d27..b972bf93d5f 100644 --- a/modules/tracking/src/mosseTracker.cpp +++ b/modules/tracking/src/mosseTracker.cpp @@ -13,24 +13,28 @@ // Cracki: for the idea of only converting the used patch to gray // -#include "opencv2/tracking.hpp" +#include "precomp.hpp" + +#include "opencv2/tracking/tracking_legacy.hpp" namespace cv { -namespace tracking { +inline namespace tracking { +namespace impl { +namespace { -struct DummyModel : TrackerModel +struct DummyModel : detail::tracking::TrackerModel { virtual void modelUpdateImpl() CV_OVERRIDE {} virtual void modelEstimationImpl( const std::vector& ) CV_OVERRIDE {} }; - const double eps=0.00001; // for normalization const double rate=0.2; // learning rate const double psrThreshold=5.7; // no detection, if PSR is smaller than this +} // namespace -struct MosseImpl CV_FINAL : TrackerMOSSE +struct MosseImpl CV_FINAL : legacy::TrackerMOSSE { protected: @@ -237,13 +241,12 @@ struct MosseImpl CV_FINAL : TrackerMOSSE }; // MosseImpl -} // tracking - +}} // namespace -Ptr TrackerMOSSE::create() +Ptr legacy::tracking::TrackerMOSSE::create() { - return makePtr(); + return makePtr(); } -} // cv +} // namespace diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 7b71bccdeb7..963e6eb6579 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -39,10 +39,17 @@ // //M*/ +#include "precomp.hpp" #include "multiTracker.hpp" -namespace cv -{ +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +namespace legacy { +inline namespace tracking { + +using namespace impl; + //Multitracker bool MultiTracker_Alt::addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm) { @@ -249,12 +256,18 @@ namespace cv return success; } +}} // namespace + + +inline namespace tracking { +namespace impl { + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, - std::vector > &trackers) + std::vector > &trackers) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[0]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + legacy::Tracker* trackerPtr = trackers[0]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); //TLD Model Extraction tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); Size initSize = tldModel->getMinSize(); @@ -445,11 +458,11 @@ namespace cv #ifdef HAVE_OPENCL void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, - std::vector > &trackers) + std::vector > &trackers) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[0]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + legacy::Tracker* trackerPtr = trackers[0]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); //TLD Model Extraction tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); Size initSize = tldModel->getMinSize(); @@ -656,4 +669,4 @@ namespace cv } #endif -} +}}} // namespace diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp index 4ab654e9ba2..504fafd22b6 100644 --- a/modules/tracking/src/multiTracker.hpp +++ b/modules/tracking/src/multiTracker.hpp @@ -42,18 +42,18 @@ #ifndef OPENCV_MULTITRACKER #define OPENCV_MULTITRACKER -#include "precomp.hpp" #include "tldTracker.hpp" #include "tldUtils.hpp" #include -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, - std::vector& detect_flgs, std::vector >& trackers); + std::vector& detect_flgs, std::vector >& trackers); #ifdef HAVE_OPENCL void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, - std::vector& detect_flgs, std::vector >& trackers); + std::vector& detect_flgs, std::vector >& trackers); #endif -} +}}} // namespace #endif \ No newline at end of file diff --git a/modules/tracking/src/multiTracker_alt.cpp b/modules/tracking/src/multiTracker_alt.cpp index 74974797721..d5a96e7cc42 100644 --- a/modules/tracking/src/multiTracker_alt.cpp +++ b/modules/tracking/src/multiTracker_alt.cpp @@ -40,8 +40,11 @@ //M*/ #include "precomp.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" namespace cv { +namespace legacy { +inline namespace tracking { // constructor MultiTracker::MultiTracker(){}; @@ -105,4 +108,4 @@ namespace cv { return makePtr(); } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/onlineBoosting.cpp b/modules/tracking/src/onlineBoosting.cpp index 3254c0b5c67..800d7ee99d7 100644 --- a/modules/tracking/src/onlineBoosting.cpp +++ b/modules/tracking/src/onlineBoosting.cpp @@ -42,8 +42,9 @@ #include "precomp.hpp" #include "opencv2/tracking/onlineBoosting.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { StrongClassifierDirectSelection::StrongClassifierDirectSelection( int numBaseClf, int numWeakClf, Size patchSz, const Rect& sampleROI, bool useFeatureEx, int iterationInit ) @@ -732,4 +733,4 @@ int ClassifierThreshold::eval( float value ) return ( ( ( m_parity * ( value - m_threshold ) ) > 0 ) ? 1 : -1 ); } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/onlineMIL.cpp b/modules/tracking/src/onlineMIL.cpp deleted file mode 100644 index 29fa9fe3499..00000000000 --- a/modules/tracking/src/onlineMIL.cpp +++ /dev/null @@ -1,381 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "precomp.hpp" -#include "opencv2/tracking/onlineMIL.hpp" - -#define sign(s) ((s > 0 ) ? 1 : ((s<0) ? -1 : 0)) - -template class SortableElementRev -{ - public: - T _val; - int _ind; - SortableElementRev() : - _ind( 0 ) - { - } - SortableElementRev( T val, int ind ) - { - _val = val; - _ind = ind; - } - bool operator<( SortableElementRev &b ) - { - return ( _val < b._val ); - } - ; -}; - -static bool CompareSortableElementRev( const SortableElementRev& i, const SortableElementRev& j ) -{ - return i._val < j._val; -} - -template void sort_order_des( std::vector &v, std::vector &order ) -{ - uint n = (uint) v.size(); - std::vector > v2; - v2.resize( n ); - order.clear(); - order.resize( n ); - for ( uint i = 0; i < n; i++ ) - { - v2[i]._ind = i; - v2[i]._val = v[i]; - } - //std::sort( v2.begin(), v2.end() ); - std::sort( v2.begin(), v2.end(), CompareSortableElementRev ); - for ( uint i = 0; i < n; i++ ) - { - order[i] = v2[i]._ind; - v[i] = v2[i]._val; - } -} -; - -namespace cv -{ - -//implementations for strong classifier - -ClfMilBoost::Params::Params() -{ - _numSel = 50; - _numFeat = 250; - _lRate = 0.85f; -} - -ClfMilBoost::ClfMilBoost() -{ - _myParams = ClfMilBoost::Params(); - _numsamples = 0; -} - -ClfMilBoost::~ClfMilBoost() -{ - _selectors.clear(); - for ( size_t i = 0; i < _weakclf.size(); i++ ) - delete _weakclf.at( i ); -} - -void ClfMilBoost::init( const ClfMilBoost::Params ¶meters ) -{ - _myParams = parameters; - _numsamples = 0; - - //_ftrs = Ftr::generate( _myParams->_ftrParams, _myParams->_numFeat ); - // if( params->_storeFtrHistory ) - // Ftr::toViz( _ftrs, "haarftrs" ); - _weakclf.resize( _myParams._numFeat ); - for ( int k = 0; k < _myParams._numFeat; k++ ) - { - _weakclf[k] = new ClfOnlineStump( k ); - _weakclf[k]->_lRate = _myParams._lRate; - - } - _counter = 0; -} - -void ClfMilBoost::update( const Mat& posx, const Mat& negx ) -{ - int numneg = negx.rows; - int numpos = posx.rows; - - // compute ftrs - //if( !posx.ftrsComputed() ) - // Ftr::compute( posx, _ftrs ); - //if( !negx.ftrsComputed() ) - // Ftr::compute( negx, _ftrs ); - - // initialize H - static std::vector Hpos, Hneg; - Hpos.clear(); - Hneg.clear(); - Hpos.resize( posx.rows, 0.0f ), Hneg.resize( negx.rows, 0.0f ); - - _selectors.clear(); - std::vector posw( posx.rows ), negw( negx.rows ); - std::vector > pospred( _weakclf.size() ), negpred( _weakclf.size() ); - - // train all weak classifiers without weights -#ifdef _OPENMP -#pragma omp parallel for -#endif - for ( int m = 0; m < _myParams._numFeat; m++ ) - { - _weakclf[m]->update( posx, negx ); - pospred[m] = _weakclf[m]->classifySetF( posx ); - negpred[m] = _weakclf[m]->classifySetF( negx ); - } - - // pick the best features - for ( int s = 0; s < _myParams._numSel; s++ ) - { - - // compute errors/likl for all weak clfs - std::vector poslikl( _weakclf.size(), 1.0f ), neglikl( _weakclf.size() ), likl( _weakclf.size() ); -#ifdef _OPENMP -#pragma omp parallel for -#endif - for ( int w = 0; w < (int) _weakclf.size(); w++ ) - { - float lll = 1.0f; - for ( int j = 0; j < numpos; j++ ) - lll *= ( 1 - sigmoid( Hpos[j] + pospred[w][j] ) ); - poslikl[w] = (float) -log( 1 - lll + 1e-5 ); - - lll = 0.0f; - for ( int j = 0; j < numneg; j++ ) - lll += (float) -log( 1e-5f + 1 - sigmoid( Hneg[j] + negpred[w][j] ) ); - neglikl[w] = lll; - - likl[w] = poslikl[w] / numpos + neglikl[w] / numneg; - } - - // pick best weak clf - std::vector order; - sort_order_des( likl, order ); - - // find best weakclf that isn't already included - for ( uint k = 0; k < order.size(); k++ ) - if( std::count( _selectors.begin(), _selectors.end(), order[k] ) == 0 ) - { - _selectors.push_back( order[k] ); - break; - } - - // update H = H + h_m -#ifdef _OPENMP -#pragma omp parallel for -#endif - for ( int k = 0; k < posx.rows; k++ ) - Hpos[k] += pospred[_selectors[s]][k]; -#ifdef _OPENMP -#pragma omp parallel for -#endif - for ( int k = 0; k < negx.rows; k++ ) - Hneg[k] += negpred[_selectors[s]][k]; - - } - - //if( _myParams->_storeFtrHistory ) - //for ( uint j = 0; j < _selectors.size(); j++ ) - // _ftrHist( _selectors[j], _counter ) = 1.0f / ( j + 1 ); - - _counter++; - /* */ - return; -} - -std::vector ClfMilBoost::classify( const Mat& x, bool logR ) -{ - int numsamples = x.rows; - std::vector res( numsamples ); - std::vector tr; - - for ( uint w = 0; w < _selectors.size(); w++ ) - { - tr = _weakclf[_selectors[w]]->classifySetF( x ); -#ifdef _OPENMP -#pragma omp parallel for -#endif - for ( int j = 0; j < numsamples; j++ ) - { - res[j] += tr[j]; - } - } - - // return probabilities or log odds ratio - if( !logR ) - { -#ifdef _OPENMP -#pragma omp parallel for -#endif - for ( int j = 0; j < (int) res.size(); j++ ) - { - res[j] = sigmoid( res[j] ); - } - } - - return res; -} - -//implementations for weak classifier - -ClfOnlineStump::ClfOnlineStump() -{ - _trained = false; - _ind = -1; - init(); -} - -ClfOnlineStump::ClfOnlineStump( int ind ) -{ - _trained = false; - _ind = ind; - init(); -} -void ClfOnlineStump::init() -{ - _mu0 = 0; - _mu1 = 0; - _sig0 = 1; - _sig1 = 1; - _lRate = 0.85f; - _trained = false; -} - -void ClfOnlineStump::update( const Mat& posx, const Mat& negx, const Mat_& /*posw*/, const Mat_& /*negw*/) -{ - //std::cout << " ClfOnlineStump::update" << _ind << std::endl; - float posmu = 0.0, negmu = 0.0; - if( posx.cols > 0 ) - posmu = float( mean( posx.col( _ind ) )[0] ); - if( negx.cols > 0 ) - negmu = float( mean( negx.col( _ind ) )[0] ); - - if( _trained ) - { - if( posx.cols > 0 ) - { - _mu1 = ( _lRate * _mu1 + ( 1 - _lRate ) * posmu ); - cv::Mat diff = posx.col( _ind ) - _mu1; - _sig1 = _lRate * _sig1 + ( 1 - _lRate ) * float( mean( diff.mul( diff ) )[0] ); - } - if( negx.cols > 0 ) - { - _mu0 = ( _lRate * _mu0 + ( 1 - _lRate ) * negmu ); - cv::Mat diff = negx.col( _ind ) - _mu0; - _sig0 = _lRate * _sig0 + ( 1 - _lRate ) * float( mean( diff.mul( diff ) )[0] ); - } - - _q = ( _mu1 - _mu0 ) / 2; - _s = sign( _mu1 - _mu0 ); - _log_n0 = std::log( float( 1.0f / pow( _sig0, 0.5f ) ) ); - _log_n1 = std::log( float( 1.0f / pow( _sig1, 0.5f ) ) ); - //_e1 = -1.0f/(2.0f*_sig1+1e-99f); - //_e0 = -1.0f/(2.0f*_sig0+1e-99f); - _e1 = -1.0f / ( 2.0f * _sig1 + std::numeric_limits::min() ); - _e0 = -1.0f / ( 2.0f * _sig0 + std::numeric_limits::min() ); - - } - else - { - _trained = true; - if( posx.cols > 0 ) - { - _mu1 = posmu; - cv::Scalar scal_mean, scal_std_dev; - cv::meanStdDev( posx.col( _ind ), scal_mean, scal_std_dev ); - _sig1 = float( scal_std_dev[0] ) * float( scal_std_dev[0] ) + 1e-9f; - } - - if( negx.cols > 0 ) - { - _mu0 = negmu; - cv::Scalar scal_mean, scal_std_dev; - cv::meanStdDev( negx.col( _ind ), scal_mean, scal_std_dev ); - _sig0 = float( scal_std_dev[0] ) * float( scal_std_dev[0] ) + 1e-9f; - } - - _q = ( _mu1 - _mu0 ) / 2; - _s = sign( _mu1 - _mu0 ); - _log_n0 = std::log( float( 1.0f / pow( _sig0, 0.5f ) ) ); - _log_n1 = std::log( float( 1.0f / pow( _sig1, 0.5f ) ) ); - //_e1 = -1.0f/(2.0f*_sig1+1e-99f); - //_e0 = -1.0f/(2.0f*_sig0+1e-99f); - _e1 = -1.0f / ( 2.0f * _sig1 + std::numeric_limits::min() ); - _e0 = -1.0f / ( 2.0f * _sig0 + std::numeric_limits::min() ); - } -} - -bool ClfOnlineStump::classify( const Mat& x, int i ) -{ - float xx = x.at( i, _ind ); - double log_p0 = ( xx - _mu0 ) * ( xx - _mu0 ) * _e0 + _log_n0; - double log_p1 = ( xx - _mu1 ) * ( xx - _mu1 ) * _e1 + _log_n1; - return log_p1 > log_p0; -} - -float ClfOnlineStump::classifyF( const Mat& x, int i ) -{ - float xx = x.at( i, _ind ); - double log_p0 = ( xx - _mu0 ) * ( xx - _mu0 ) * _e0 + _log_n0; - double log_p1 = ( xx - _mu1 ) * ( xx - _mu1 ) * _e1 + _log_n1; - return float( log_p1 - log_p0 ); -} - -inline std::vector ClfOnlineStump::classifySetF( const Mat& x ) -{ - std::vector res( x.rows ); - -#ifdef _OPENMP -#pragma omp parallel for -#endif - for ( int k = 0; k < (int) res.size(); k++ ) - { - res[k] = classifyF( x, k ); - } - return res; -} - -} /* namespace cv */ diff --git a/modules/tracking/src/precomp.hpp b/modules/tracking/src/precomp.hpp index f892f867832..50ebd727224 100644 --- a/modules/tracking/src/precomp.hpp +++ b/modules/tracking/src/precomp.hpp @@ -42,18 +42,29 @@ #ifndef __OPENCV_PRECOMP_H__ #define __OPENCV_PRECOMP_H__ -#include "cvconfig.h" -#include "opencv2/tracking.hpp" -#include "opencv2/core/utility.hpp" +#include "opencv2/core.hpp" #include "opencv2/core/ocl.hpp" -#include #include "opencv2/core/hal/hal.hpp" -namespace cv -{ - extern const float ColorNames[][10]; +#include "opencv2/video/tracking.hpp" + +#include "opencv2/tracking.hpp" + - namespace tracking { +#include "opencv2/tracking/tracking_internals.hpp" + +namespace cv { inline namespace tracking { +namespace impl { } +using namespace impl; +using namespace cv::detail::tracking; +}} // namespace + + +namespace cv { +namespace detail { +inline namespace tracking { + + extern const float ColorNames[][10]; /* Cholesky decomposition The function performs Cholesky decomposition . @@ -102,7 +113,6 @@ namespace cv return success; } - } // tracking -} // cv +}}} // namespace #endif diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 97b29645fec..6ce0a9785b6 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -39,10 +39,13 @@ // //M*/ +#include "precomp.hpp" #include "opencv2/tracking/tldDataset.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { + namespace tld { char tldRootPath[100]; @@ -182,4 +185,5 @@ namespace cv } } -} + +}}} diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index bb52a4899e3..82688213551 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -39,15 +39,15 @@ // //M*/ +#include "precomp.hpp" + #include "tldDetector.hpp" #include "tracking_utils.hpp" -#include - -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { // Calculate offsets for classifiers void TLDDetector::prepareClassifiers(int rowstep) { @@ -619,5 +619,4 @@ namespace cv return ((p2 - p * p) > VARIANCE_THRESHOLD * *originalVariance); } - } -} +}}}} // namespace diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 292cbbb356d..a2b329d6e49 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -42,15 +42,15 @@ #ifndef OPENCV_TLD_DETECTOR #define OPENCV_TLD_DETECTOR -#include "precomp.hpp" #include "opencl_kernels_tracking.hpp" #include "tldEnsembleClassifier.hpp" #include "tldUtils.hpp" -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + const int STANDARD_PATCH_SIZE = 15; const int NEG_EXAMPLES_IN_INIT_MODEL = 300; const int MAX_EXAMPLES_IN_MODEL = 500; @@ -116,7 +116,6 @@ namespace cv }; - } -} +}}}} // namespace #endif diff --git a/modules/tracking/src/tldEnsembleClassifier.cpp b/modules/tracking/src/tldEnsembleClassifier.cpp index 1d72b3e81d4..c800fc3c9fc 100644 --- a/modules/tracking/src/tldEnsembleClassifier.cpp +++ b/modules/tracking/src/tldEnsembleClassifier.cpp @@ -39,12 +39,14 @@ // //M*/ +#include "precomp.hpp" #include "tldEnsembleClassifier.hpp" -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + // Constructor TLDEnsembleClassifier::TLDEnsembleClassifier(const std::vector& meas, int beg, int end) :lastStep_(-1) { @@ -194,5 +196,4 @@ namespace cv return (int)classifiers.size(); } - } -} \ No newline at end of file +}}}} // namespace diff --git a/modules/tracking/src/tldEnsembleClassifier.hpp b/modules/tracking/src/tldEnsembleClassifier.hpp index f0ec175bad7..6df65e4814d 100644 --- a/modules/tracking/src/tldEnsembleClassifier.hpp +++ b/modules/tracking/src/tldEnsembleClassifier.hpp @@ -40,12 +40,12 @@ //M*/ #include -#include "precomp.hpp" -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + class TLDEnsembleClassifier { public: @@ -64,5 +64,5 @@ namespace cv std::vector offset; int lastStep_; }; - } -} \ No newline at end of file + +}}}} // namespace diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 833e586d0c8..ad787ab8e01 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -39,14 +39,14 @@ // //M*/ +#include "precomp.hpp" #include "tldModel.hpp" -#include +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { -namespace cv -{ - namespace tld - { //Constructor TrackerTLDModel::TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize): timeStampPositiveNext(0), timeStampNegativeNext(0), minSize_(minSize), params_(params), boundingBox_(boundingBox) @@ -361,5 +361,5 @@ namespace cv dfprintf((port, "\tpositiveExamples.size() = %d\n", (int)positiveExamples.size())); dfprintf((port, "\tnegativeExamples.size() = %d\n", (int)negativeExamples.size())); } - } -} + +}}}} // namespace diff --git a/modules/tracking/src/tldModel.hpp b/modules/tracking/src/tldModel.hpp index 7616ebdc439..57331756cb0 100644 --- a/modules/tracking/src/tldModel.hpp +++ b/modules/tracking/src/tldModel.hpp @@ -45,10 +45,15 @@ #include "tldDetector.hpp" #include "tldUtils.hpp" -namespace cv -{ - namespace tld - { +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + +using namespace cv::legacy; + class TrackerTLDModel : public TrackerModel { public: @@ -84,7 +89,6 @@ namespace cv RNG rng; }; - } -} +}}}} // namespace #endif diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index db1fccf5498..92038ef2d5e 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -39,10 +39,15 @@ // //M*/ +#include "precomp.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" #include "tldTracker.hpp" -namespace cv -{ +namespace cv { +namespace legacy { +inline namespace tracking { +using namespace impl; +using namespace impl::tld; TrackerTLD::Params::Params(){} @@ -60,8 +65,11 @@ Ptr TrackerTLD::create() return Ptr(new tld::TrackerTLDImpl()); } -namespace tld -{ +}} // namespace + +inline namespace tracking { +namespace impl { +namespace tld { TrackerTLDImpl::TrackerTLDImpl(const TrackerTLD::Params ¶meters) : params( parameters ) @@ -323,6 +331,4 @@ void Data::printme(FILE* port) dfprintf((port, "\tminSize = %dx%d\n", minSize.width, minSize.height)); } -} - -} +}}}} // namespace diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index 3f9861192cc..e96b3e04d34 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -42,18 +42,17 @@ #ifndef OPENCV_TLD_TRACKER #define OPENCV_TLD_TRACKER -#include "precomp.hpp" #include "opencv2/video/tracking.hpp" #include "opencv2/imgproc.hpp" #include "tldModel.hpp" -#include -#include +#include +#include -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { -namespace tld -{ class TrackerProxy { public: @@ -168,7 +167,6 @@ class TrackerTLDImpl : public TrackerTLD }; -} -} +}}}} // namespace #endif diff --git a/modules/tracking/src/tldUtils.cpp b/modules/tracking/src/tldUtils.cpp index b570a1f3bb9..5dca7b5f855 100644 --- a/modules/tracking/src/tldUtils.cpp +++ b/modules/tracking/src/tldUtils.cpp @@ -39,13 +39,14 @@ // //M*/ +#include "precomp.hpp" #include "tldUtils.hpp" -namespace cv -{ -namespace tld -{ +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { //Debug functions and variables Rect2d etalon(14.0, 110.0, 20.0, 20.0); @@ -192,4 +193,4 @@ void resample(const Mat& img, const Rect2d& r2, Mat_& samples) } -}} +}}}} // namespace diff --git a/modules/tracking/src/tldUtils.hpp b/modules/tracking/src/tldUtils.hpp index 1a606429d28..311e08f5ed4 100644 --- a/modules/tracking/src/tldUtils.hpp +++ b/modules/tracking/src/tldUtils.hpp @@ -1,12 +1,11 @@ #ifndef OPENCV_TLD_UTILS #define OPENCV_TLD_UTILS -#include "precomp.hpp" +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { -namespace cv -{ - namespace tld - { //debug functions and variables #define ALEX_DEBUG #ifdef ALEX_DEBUG @@ -48,7 +47,7 @@ namespace cv double variance(const Mat& img); void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector& res); double scaleAndBlur(const Mat& originalImg, int scale, Mat& scaledImg, Mat& blurredImg, Size GaussBlurKernelSize, double scaleStep); - } -} + +}}}} // namespace #endif diff --git a/modules/tracking/src/tracker.cpp b/modules/tracking/src/tracker.cpp index f692b5d62f9..42c9ccaa69a 100644 --- a/modules/tracking/src/tracker.cpp +++ b/modules/tracking/src/tracker.cpp @@ -1,100 +1,9 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. #include "precomp.hpp" -namespace cv -{ +// see modules/video/src/tracking/tracker.cpp -/* - * Tracker - */ - -Tracker::~Tracker() -{ -} - -bool Tracker::init( InputArray image, const Rect2d& boundingBox ) -{ - - if( isInit ) - { - return false; - } - - if( image.empty() ) - return false; - - sampler = Ptr( new TrackerSampler() ); - featureSet = Ptr( new TrackerFeatureSet() ); - model = Ptr(); - - bool initTracker = initImpl( image.getMat(), boundingBox ); - - //check if the model component is initialized - if (!model) - { - CV_Error( -1, "The model is not initialized" ); - } - - if( initTracker ) - { - isInit = true; - } - - return initTracker; -} - -bool Tracker::update( InputArray image, Rect2d& boundingBox ) -{ - - if( !isInit ) - { - return false; - } - - if( image.empty() ) - return false; - - return updateImpl( image.getMat(), boundingBox ); -} - -} /* namespace cv */ +#include "legacy/tracker.legacy.hpp" diff --git a/modules/tracking/src/trackerBoosting.cpp b/modules/tracking/src/trackerBoosting.cpp index 05b11aa2d59..9536e8db235 100644 --- a/modules/tracking/src/trackerBoosting.cpp +++ b/modules/tracking/src/trackerBoosting.cpp @@ -42,8 +42,12 @@ #include "precomp.hpp" #include "trackerBoostingModel.hpp" -namespace cv -{ +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +namespace legacy { +inline namespace tracking { +using namespace impl; class TrackerBoostingImpl : public TrackerBoosting { @@ -132,7 +136,7 @@ bool TrackerBoostingImpl::initImpl( const Mat& image, const Rect2d& boundingBox CSparameters.overlap = params.samplerOverlap; CSparameters.searchFactor = params.samplerSearchFactor; - Ptr CSSampler = Ptr( new TrackerSamplerCS( CSparameters ) ); + Ptr CSSampler = Ptr( new TrackerSamplerCS( CSparameters ) ); if( !sampler->addTrackerSamplerAlgorithm( CSSampler ) ) return false; @@ -151,11 +155,11 @@ bool TrackerBoostingImpl::initImpl( const Mat& image, const Rect2d& boundingBox Rect ROI = CSSampler.staticCast()->getROI(); //compute HAAR features - TrackerFeatureHAAR::Params HAARparameters; + TrackerContribFeatureHAAR::Params HAARparameters; HAARparameters.numFeatures = params.featureSetNumFeatures; HAARparameters.isIntegral = true; HAARparameters.rectSize = Size( static_cast(boundingBox.width), static_cast(boundingBox.height) ); - Ptr trackerFeature = Ptr( new TrackerFeatureHAAR( HAARparameters ) ); + Ptr trackerFeature = Ptr( new TrackerContribFeatureHAAR( HAARparameters ) ); if( !featureSet->addTrackerFeature( trackerFeature ) ) return false; @@ -175,11 +179,11 @@ bool TrackerBoostingImpl::initImpl( const Mat& image, const Rect2d& boundingBox for ( int i = 0; i < params.iterationInit; i++ ) { //compute temp features - TrackerFeatureHAAR::Params HAARparameters2; + TrackerContribFeatureHAAR::Params HAARparameters2; HAARparameters2.numFeatures = static_cast( posSamples.size() + negSamples.size() ); HAARparameters2.isIntegral = true; HAARparameters2.rectSize = Size( static_cast(boundingBox.width), static_cast(boundingBox.height) ); - Ptr trackerFeature2 = Ptr( new TrackerFeatureHAAR( HAARparameters2 ) ); + Ptr trackerFeature2 = Ptr( new TrackerContribFeatureHAAR( HAARparameters2 ) ); model.staticCast()->setMode( TrackerBoostingModel::MODE_NEGATIVE, negSamples ); model->modelEstimation( negResponse ); @@ -194,8 +198,8 @@ bool TrackerBoostingImpl::initImpl( const Mat& image, const Rect2d& boundingBox { if( replacedClassifier[j] != -1 && swappedClassified[j] != -1 ) { - trackerFeature.staticCast()->swapFeature( replacedClassifier[j], swappedClassified[j] ); - trackerFeature.staticCast()->swapFeature( swappedClassified[j], trackerFeature2->getFeatureAt( (int)j ) ); + trackerFeature.staticCast()->swapFeature( replacedClassifier[j], swappedClassified[j] ); + trackerFeature.staticCast()->swapFeature( swappedClassified[j], trackerFeature2->getFeatureAt( (int)j ) ); } } } @@ -240,7 +244,7 @@ bool TrackerBoostingImpl::updateImpl( const Mat& image, Rect2d& boundingBox ) Mat response; std::vector classifiers = model->getTrackerStateEstimator().staticCast()->computeSelectedWeakClassifier(); - Ptr extractor = featureSet->getTrackerFeature()[0].second.staticCast(); + Ptr extractor = featureSet->getTrackerFeature()[0].second.staticCast(); extractor->extractSelected( classifiers, detectSamples, response ); responses.push_back( response ); @@ -288,11 +292,11 @@ bool TrackerBoostingImpl::updateImpl( const Mat& image, Rect2d& boundingBox ) const std::vector negResponse = featureSet->getResponses(); //compute temp features - TrackerFeatureHAAR::Params HAARparameters2; + TrackerContribFeatureHAAR::Params HAARparameters2; HAARparameters2.numFeatures = static_cast( posSamples.size() + negSamples.size() ); HAARparameters2.isIntegral = true; HAARparameters2.rectSize = Size( static_cast(boundingBox.width), static_cast(boundingBox.height) ); - Ptr trackerFeature2 = Ptr( new TrackerFeatureHAAR( HAARparameters2 ) ); + Ptr trackerFeature2 = Ptr( new TrackerContribFeatureHAAR( HAARparameters2 ) ); //model estimate model.staticCast()->setMode( TrackerBoostingModel::MODE_NEGATIVE, negSamples ); @@ -310,8 +314,8 @@ bool TrackerBoostingImpl::updateImpl( const Mat& image, Rect2d& boundingBox ) { if( replacedClassifier[j] != -1 && swappedClassified[j] != -1 ) { - featureSet->getTrackerFeature().at( 0 ).second.staticCast()->swapFeature( replacedClassifier[j], swappedClassified[j] ); - featureSet->getTrackerFeature().at( 0 ).second.staticCast()->swapFeature( swappedClassified[j], + featureSet->getTrackerFeature().at( 0 ).second.staticCast()->swapFeature( replacedClassifier[j], swappedClassified[j] ); + featureSet->getTrackerFeature().at( 0 ).second.staticCast()->swapFeature( swappedClassified[j], trackerFeature2->getFeatureAt( (int)j ) ); } } @@ -319,4 +323,4 @@ bool TrackerBoostingImpl::updateImpl( const Mat& image, Rect2d& boundingBox ) return true; } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/trackerBoostingModel.cpp b/modules/tracking/src/trackerBoostingModel.cpp index bd6148a9dbf..3fb48f0959e 100644 --- a/modules/tracking/src/trackerBoostingModel.cpp +++ b/modules/tracking/src/trackerBoostingModel.cpp @@ -39,14 +39,16 @@ // //M*/ +#include "precomp.hpp" #include "trackerBoostingModel.hpp" /** * TrackerBoostingModel */ -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { TrackerBoostingModel::TrackerBoostingModel( const Rect& boundingBox ) { @@ -119,4 +121,4 @@ void TrackerBoostingModel::responseToConfidenceMap( const std::vector& resp } } -} +}}} // namespace diff --git a/modules/tracking/src/trackerBoostingModel.hpp b/modules/tracking/src/trackerBoostingModel.hpp index 0d4582cd16a..45aa1dc6dfb 100644 --- a/modules/tracking/src/trackerBoostingModel.hpp +++ b/modules/tracking/src/trackerBoostingModel.hpp @@ -42,11 +42,9 @@ #ifndef __OPENCV_TRACKER_BOOSTING_MODEL_HPP__ #define __OPENCV_TRACKER_BOOSTING_MODEL_HPP__ -#include "precomp.hpp" -#include "opencv2/core.hpp" - -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { /** * \brief Implementation of TrackerModel for BOOSTING algorithm @@ -103,6 +101,6 @@ class TrackerBoostingModel : public TrackerModel int mode; }; -} /* namespace cv */ +}}} // namespace #endif diff --git a/modules/tracking/src/trackerCSRT.cpp b/modules/tracking/src/trackerCSRT.cpp index 900fea51a64..1b3f48aa714 100644 --- a/modules/tracking/src/trackerCSRT.cpp +++ b/modules/tracking/src/trackerCSRT.cpp @@ -8,35 +8,38 @@ #include "trackerCSRTUtils.hpp" #include "trackerCSRTScaleEstimation.hpp" -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { + /** * \brief Implementation of TrackerModel for CSRT algorithm */ -class TrackerCSRTModel : public TrackerModel +class TrackerCSRTModel CV_FINAL : public TrackerModel { public: - TrackerCSRTModel(TrackerCSRT::Params /*params*/){} + TrackerCSRTModel(){} ~TrackerCSRTModel(){} protected: void modelEstimationImpl(const std::vector& /*responses*/) CV_OVERRIDE {} void modelUpdateImpl() CV_OVERRIDE {} }; - -class TrackerCSRTImpl : public TrackerCSRT +class TrackerCSRTImpl CV_FINAL : public TrackerCSRT { public: - TrackerCSRTImpl(const TrackerCSRT::Params ¶meters = TrackerCSRT::Params()); - void read(const FileNode& fn) CV_OVERRIDE; - void write(FileStorage& fs) const CV_OVERRIDE; + TrackerCSRTImpl(const Params ¶meters = Params()); -protected: - TrackerCSRT::Params params; + Params params; - bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE; + Ptr model; + + // Tracker API + virtual void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + virtual bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; virtual void setInitialMask(InputArray mask) CV_OVERRIDE; - bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE; + +protected: void update_csr_filter(const Mat &image, const Mat &my_mask); void update_histograms(const Mat &image, const Rect ®ion); void extract_histograms(const Mat &image, cv::Rect region, Histogram &hf, Histogram &hb); @@ -49,7 +52,6 @@ class TrackerCSRTImpl : public TrackerCSRT Point2f estimate_new_position(const Mat &image); std::vector get_features(const Mat &patch, const Size2i &feature_size); -private: bool check_mask_area(const Mat &mat, const double obj_area); float current_scale_factor; Mat window; @@ -75,28 +77,10 @@ class TrackerCSRTImpl : public TrackerCSRT int cell_size; }; -Ptr TrackerCSRT::create(const TrackerCSRT::Params ¶meters) -{ - return Ptr(new TrackerCSRTImpl(parameters)); -} -Ptr TrackerCSRT::create() -{ - return Ptr(new TrackerCSRTImpl()); -} TrackerCSRTImpl::TrackerCSRTImpl(const TrackerCSRT::Params ¶meters) : params(parameters) { - isInit = false; -} - -void TrackerCSRTImpl::read(const cv::FileNode& fn) -{ - params.read(fn); -} - -void TrackerCSRTImpl::write(cv::FileStorage& fs) const -{ - params.write(fs); + // nothing } void TrackerCSRTImpl::setInitialMask(InputArray mask) @@ -463,13 +447,13 @@ Point2f TrackerCSRTImpl::estimate_new_position(const Mat &image) // ********************************************************************* // * Update API function * // ********************************************************************* -bool TrackerCSRTImpl::updateImpl(const Mat& image_, Rect2d& boundingBox) +bool TrackerCSRTImpl::update(InputArray image_, Rect& boundingBox) { Mat image; if(image_.channels() == 1) //treat gray image as color image cvtColor(image_, image, COLOR_GRAY2BGR); else - image = image_; + image = image_.getMat(); object_center = estimate_new_position(image); if (object_center.x < 0 && object_center.y < 0) @@ -506,13 +490,13 @@ bool TrackerCSRTImpl::updateImpl(const Mat& image_, Rect2d& boundingBox) // ********************************************************************* // * Init API function * // ********************************************************************* -bool TrackerCSRTImpl::initImpl(const Mat& image_, const Rect2d& boundingBox) +void TrackerCSRTImpl::init(InputArray image_, const Rect& boundingBox) { Mat image; if(image_.channels() == 1) //treat gray image as color image cvtColor(image_, image, COLOR_GRAY2BGR); else - image = image_; + image = image_.getMat(); current_scale_factor = 1.0; image_size = image.size(); @@ -545,8 +529,7 @@ bool TrackerCSRTImpl::initImpl(const Mat& image_, const Rect2d& boundingBox) } else if(params.window_function.compare("kaiser") == 0) { window = get_kaiser_win(Size(yf.cols,yf.rows), params.kaiser_alpha); } else { - std::cout << "Not a valid window function" << std::endl; - return false; + CV_Error(Error::StsBadArg, "Not a valid window function"); } Size2i scaled_obj_size = Size2i(cvFloor(original_target_size.width * rescale_ratio / cell_size), @@ -616,11 +599,11 @@ bool TrackerCSRTImpl::initImpl(const Mat& image_, const Rect2d& boundingBox) dsst = DSST(image, bounding_box, template_size, params.number_of_scales, params.scale_step, params.scale_model_max_area, params.scale_sigma_factor, params.scale_lr); - model=Ptr(new TrackerCSRTModel(params)); - isInit = true; - return true; + model=makePtr(); } +} // namespace impl + TrackerCSRT::Params::Params() { use_channel_weights = true; @@ -652,94 +635,21 @@ TrackerCSRT::Params::Params() psr_threshold = 0.035f; } -void TrackerCSRT::Params::read(const FileNode& fn) +TrackerCSRT::TrackerCSRT() { - *this = TrackerCSRT::Params(); - if(!fn["padding"].empty()) - fn["padding"] >> padding; - if(!fn["template_size"].empty()) - fn["template_size"] >> template_size; - if(!fn["gsl_sigma"].empty()) - fn["gsl_sigma"] >> gsl_sigma; - if(!fn["hog_orientations"].empty()) - fn["hog_orientations"] >> hog_orientations; - if(!fn["num_hog_channels_used"].empty()) - fn["num_hog_channels_used"] >> num_hog_channels_used; - if(!fn["hog_clip"].empty()) - fn["hog_clip"] >> hog_clip; - if(!fn["use_hog"].empty()) - fn["use_hog"] >> use_hog; - if(!fn["use_color_names"].empty()) - fn["use_color_names"] >> use_color_names; - if(!fn["use_gray"].empty()) - fn["use_gray"] >> use_gray; - if(!fn["use_rgb"].empty()) - fn["use_rgb"] >> use_rgb; - if(!fn["window_function"].empty()) - fn["window_function"] >> window_function; - if(!fn["kaiser_alpha"].empty()) - fn["kaiser_alpha"] >> kaiser_alpha; - if(!fn["cheb_attenuation"].empty()) - fn["cheb_attenuation"] >> cheb_attenuation; - if(!fn["filter_lr"].empty()) - fn["filter_lr"] >> filter_lr; - if(!fn["admm_iterations"].empty()) - fn["admm_iterations"] >> admm_iterations; - if(!fn["number_of_scales"].empty()) - fn["number_of_scales"] >> number_of_scales; - if(!fn["scale_sigma_factor"].empty()) - fn["scale_sigma_factor"] >> scale_sigma_factor; - if(!fn["scale_model_max_area"].empty()) - fn["scale_model_max_area"] >> scale_model_max_area; - if(!fn["scale_lr"].empty()) - fn["scale_lr"] >> scale_lr; - if(!fn["scale_step"].empty()) - fn["scale_step"] >> scale_step; - if(!fn["use_channel_weights"].empty()) - fn["use_channel_weights"] >> use_channel_weights; - if(!fn["weights_lr"].empty()) - fn["weights_lr"] >> weights_lr; - if(!fn["use_segmentation"].empty()) - fn["use_segmentation"] >> use_segmentation; - if(!fn["histogram_bins"].empty()) - fn["histogram_bins"] >> histogram_bins; - if(!fn["background_ratio"].empty()) - fn["background_ratio"] >> background_ratio; - if(!fn["histogram_lr"].empty()) - fn["histogram_lr"] >> histogram_lr; - if(!fn["psr_threshold"].empty()) - fn["psr_threshold"] >> psr_threshold; - CV_Assert(number_of_scales % 2 == 1); - CV_Assert(use_gray || use_color_names || use_hog || use_rgb); + // nothing } -void TrackerCSRT::Params::write(FileStorage& fs) const + +TrackerCSRT::~TrackerCSRT() { - fs << "padding" << padding; - fs << "template_size" << template_size; - fs << "gsl_sigma" << gsl_sigma; - fs << "hog_orientations" << hog_orientations; - fs << "num_hog_channels_used" << num_hog_channels_used; - fs << "hog_clip" << hog_clip; - fs << "use_hog" << use_hog; - fs << "use_color_names" << use_color_names; - fs << "use_gray" << use_gray; - fs << "use_rgb" << use_rgb; - fs << "window_function" << window_function; - fs << "kaiser_alpha" << kaiser_alpha; - fs << "cheb_attenuation" << cheb_attenuation; - fs << "filter_lr" << filter_lr; - fs << "admm_iterations" << admm_iterations; - fs << "number_of_scales" << number_of_scales; - fs << "scale_sigma_factor" << scale_sigma_factor; - fs << "scale_model_max_area" << scale_model_max_area; - fs << "scale_lr" << scale_lr; - fs << "scale_step" << scale_step; - fs << "use_channel_weights" << use_channel_weights; - fs << "weights_lr" << weights_lr; - fs << "use_segmentation" << use_segmentation; - fs << "histogram_bins" << histogram_bins; - fs << "background_ratio" << background_ratio; - fs << "histogram_lr" << histogram_lr; - fs << "psr_threshold" << psr_threshold; + // nothing } -} /* namespace cv */ + +Ptr TrackerCSRT::create(const TrackerCSRT::Params ¶meters) +{ + return makePtr(parameters); +} + +}} // namespace + +#include "legacy/trackerCSRT.legacy.hpp" diff --git a/modules/tracking/src/trackerCSRTScaleEstimation.cpp b/modules/tracking/src/trackerCSRTScaleEstimation.cpp index 5bde5f420b7..221d1ee6413 100644 --- a/modules/tracking/src/trackerCSRTScaleEstimation.cpp +++ b/modules/tracking/src/trackerCSRTScaleEstimation.cpp @@ -127,7 +127,7 @@ DSST::DSST(const Mat &image, mulSpectrums(ysf, Fscale_resp, sf_num, 0 , true); Mat sf_den_all; mulSpectrums(Fscale_resp, Fscale_resp, sf_den_all, 0, true); - reduce(sf_den_all, sf_den, 0, CV_REDUCE_SUM, -1); + reduce(sf_den_all, sf_den, 0, REDUCE_SUM, -1); } DSST::~DSST() @@ -178,7 +178,7 @@ void DSST::update(const Mat &image, const Point2f object_center) mulSpectrums(ysf, Fscale_features, new_sf_num, DFT_ROWS, true); Mat sf_den_all; mulSpectrums(Fscale_features, Fscale_features, new_sf_den_all, DFT_ROWS, true); - reduce(new_sf_den_all, new_sf_den, 0, CV_REDUCE_SUM, -1); + reduce(new_sf_den_all, new_sf_den, 0, REDUCE_SUM, -1); sf_num = (1 - learn_rate) * sf_num + learn_rate * new_sf_num; sf_den = (1 - learn_rate) * sf_den + learn_rate * new_sf_den; @@ -194,7 +194,7 @@ float DSST::getScale(const Mat &image, const Point2f object_center) mulSpectrums(Fscale_features, sf_num, Fscale_features, 0, false); Mat scale_resp; - reduce(Fscale_features, scale_resp, 0, CV_REDUCE_SUM, -1); + reduce(Fscale_features, scale_resp, 0, REDUCE_SUM, -1); scale_resp = divide_complex_matrices(scale_resp, sf_den + 0.01f); idft(scale_resp, scale_resp, DFT_REAL_OUTPUT|DFT_SCALE); Point max_loc; diff --git a/modules/tracking/src/trackerFeature.cpp b/modules/tracking/src/trackerFeature.cpp index a5b59bdcfa1..e216d5873ba 100644 --- a/modules/tracking/src/trackerFeature.cpp +++ b/modules/tracking/src/trackerFeature.cpp @@ -41,27 +41,20 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* - * TrackerFeature + * TrackerContribFeature */ -TrackerFeature::~TrackerFeature() -{ - -} - -void TrackerFeature::compute( const std::vector& images, Mat& response ) +TrackerContribFeature::~TrackerContribFeature() { - if( images.empty() ) - return; - computeImpl( images, response ); } -Ptr TrackerFeature::create( const String& trackerFeatureType ) +Ptr TrackerContribFeature::create( const String& trackerFeatureType ) { if( trackerFeatureType.find( "FEATURE2D" ) == 0 ) { @@ -81,7 +74,7 @@ Ptr TrackerFeature::create( const String& trackerFeatureType ) if( trackerFeatureType.find( "HAAR" ) == 0 ) { - return Ptr( new TrackerFeatureHAAR() ); + return Ptr( new TrackerContribFeatureHAAR() ); } if( trackerFeatureType.find( "LBP" ) == 0 ) @@ -92,7 +85,7 @@ Ptr TrackerFeature::create( const String& trackerFeatureType ) CV_Error( -1, "Tracker feature type not supported" ); } -String TrackerFeature::getClassName() const +String TrackerContribFeature::getClassName() const { return className; } @@ -144,21 +137,21 @@ void TrackerFeatureHOG::selection( Mat& /*response*/, int /*npoints*/) } /** - * TrackerFeatureHAAR + * TrackerContribFeatureHAAR */ /** * Parameters */ -TrackerFeatureHAAR::Params::Params() +TrackerContribFeatureHAAR::Params::Params() { numFeatures = 250; rectSize = Size( 100, 100 ); isIntegral = false; } -TrackerFeatureHAAR::TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶meters ) : +TrackerContribFeatureHAAR::TrackerContribFeatureHAAR( const TrackerContribFeatureHAAR::Params ¶meters ) : params( parameters ) { className = "HAAR"; @@ -170,23 +163,23 @@ TrackerFeatureHAAR::TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶me featureEvaluator->init( &haarParams, 1, params.rectSize ); } -TrackerFeatureHAAR::~TrackerFeatureHAAR() +TrackerContribFeatureHAAR::~TrackerContribFeatureHAAR() { } -CvHaarEvaluator::FeatureHaar& TrackerFeatureHAAR::getFeatureAt( int id ) +CvHaarEvaluator::FeatureHaar& TrackerContribFeatureHAAR::getFeatureAt( int id ) { return featureEvaluator->getFeatures( id ); } -bool TrackerFeatureHAAR::swapFeature( int id, CvHaarEvaluator::FeatureHaar& feature ) +bool TrackerContribFeatureHAAR::swapFeature( int id, CvHaarEvaluator::FeatureHaar& feature ) { featureEvaluator->getFeatures( id ) = feature; return true; } -bool TrackerFeatureHAAR::swapFeature( int source, int target ) +bool TrackerContribFeatureHAAR::swapFeature( int source, int target ) { CvHaarEvaluator::FeatureHaar feature = featureEvaluator->getFeatures( source ); featureEvaluator->getFeatures( source ) = featureEvaluator->getFeatures( target ); @@ -194,7 +187,7 @@ bool TrackerFeatureHAAR::swapFeature( int source, int target ) return true; } -bool TrackerFeatureHAAR::extractSelected( const std::vector selFeatures, const std::vector& images, Mat& response ) +bool TrackerContribFeatureHAAR::extractSelected( const std::vector selFeatures, const std::vector& images, Mat& response ) { if( images.empty() ) { @@ -263,7 +256,7 @@ class Parallel_compute : public cv::ParallelLoopBody } }; -bool TrackerFeatureHAAR::computeImpl( const std::vector& images, Mat& response ) +bool TrackerContribFeatureHAAR::computeImpl( const std::vector& images, Mat& response ) { if( images.empty() ) { @@ -293,7 +286,7 @@ bool TrackerFeatureHAAR::computeImpl( const std::vector& images, Mat& respo return true; } -void TrackerFeatureHAAR::selection( Mat& /*response*/, int /*npoints*/) +void TrackerContribFeatureHAAR::selection( Mat& /*response*/, int /*npoints*/) { } @@ -321,4 +314,4 @@ void TrackerFeatureLBP::selection( Mat& /*response*/, int /*npoints*/) } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/trackerFeatureSet.cpp b/modules/tracking/src/trackerFeatureSet.cpp index 9896f0ffa29..e391ea9f9aa 100644 --- a/modules/tracking/src/trackerFeatureSet.cpp +++ b/modules/tracking/src/trackerFeatureSet.cpp @@ -41,17 +41,18 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* - * TrackerFeatureSet + * TrackerContribFeatureSet */ /* * Constructor */ -TrackerFeatureSet::TrackerFeatureSet() +TrackerContribFeatureSet::TrackerContribFeatureSet() { blockAddTrackerFeature = false; } @@ -59,12 +60,12 @@ TrackerFeatureSet::TrackerFeatureSet() /* * Destructor */ -TrackerFeatureSet::~TrackerFeatureSet() +TrackerContribFeatureSet::~TrackerContribFeatureSet() { } -void TrackerFeatureSet::extraction( const std::vector& images ) +void TrackerContribFeatureSet::extraction( const std::vector& images ) { clearResponses(); @@ -83,23 +84,23 @@ void TrackerFeatureSet::extraction( const std::vector& images ) } } -void TrackerFeatureSet::selection() +void TrackerContribFeatureSet::selection() { } -void TrackerFeatureSet::removeOutliers() +void TrackerContribFeatureSet::removeOutliers() { } -bool TrackerFeatureSet::addTrackerFeature( String trackerFeatureType ) +bool TrackerContribFeatureSet::addTrackerFeature( String trackerFeatureType ) { if( blockAddTrackerFeature ) { return false; } - Ptr feature = TrackerFeature::create( trackerFeatureType ); + Ptr feature = TrackerContribFeature::create( trackerFeatureType ); if (!feature) { @@ -111,7 +112,7 @@ bool TrackerFeatureSet::addTrackerFeature( String trackerFeatureType ) return true; } -bool TrackerFeatureSet::addTrackerFeature( Ptr& feature ) +bool TrackerContribFeatureSet::addTrackerFeature( Ptr& feature ) { if( blockAddTrackerFeature ) { @@ -124,19 +125,20 @@ bool TrackerFeatureSet::addTrackerFeature( Ptr& feature ) return true; } -const std::vector > >& TrackerFeatureSet::getTrackerFeature() const +const std::vector > >& TrackerContribFeatureSet::getTrackerFeature() const { return features; } -const std::vector& TrackerFeatureSet::getResponses() const +const std::vector& TrackerContribFeatureSet::getResponses() const { return responses; } -void TrackerFeatureSet::clearResponses() +void TrackerContribFeatureSet::clearResponses() { responses.clear(); } -} /* namespace cv */ + +}}} // namespace diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 471ad71f7aa..9623f2f1b08 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -40,55 +40,50 @@ //M*/ #include "precomp.hpp" + #include "opencl_kernels_tracking.hpp" #include #include +namespace cv { +inline namespace tracking { +namespace impl { + /*--------------------------- | TrackerKCFModel |---------------------------*/ -namespace cv{ /** * \brief Implementation of TrackerModel for KCF algorithm */ class TrackerKCFModel : public TrackerModel{ public: - TrackerKCFModel(TrackerKCF::Params /*params*/){} + TrackerKCFModel(){} ~TrackerKCFModel(){} protected: void modelEstimationImpl( const std::vector& /*responses*/ ) CV_OVERRIDE {} void modelUpdateImpl() CV_OVERRIDE {} }; -} /* namespace cv */ /*--------------------------- | TrackerKCF |---------------------------*/ -namespace cv{ - /* * Prototype */ - class TrackerKCFImpl : public TrackerKCF { - public: - TrackerKCFImpl( const TrackerKCF::Params ¶meters = TrackerKCF::Params() ); - void read( const FileNode& /*fn*/ ) CV_OVERRIDE; - void write( FileStorage& /*fs*/ ) const CV_OVERRIDE; - void setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func = false) CV_OVERRIDE; +class TrackerKCFImpl CV_FINAL : public TrackerKCF +{ +public: + TrackerKCFImpl(const TrackerKCF::Params ¶meters); - protected: - /* - * basic functions and vars - */ - bool initImpl( const Mat& /*image*/, const Rect2d& boundingBox ) CV_OVERRIDE; - bool updateImpl( const Mat& image, Rect2d& boundingBox ) CV_OVERRIDE; + virtual void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + virtual bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; + void setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func = false) CV_OVERRIDE; TrackerKCF::Params params; + Ptr model; - /* - * KCF functions and vars - */ +protected: void createHanningWindow(OutputArray dest, const cv::Size winSize, const int type) const; void inline fft2(const Mat src, std::vector & dest, std::vector & layers_data) const; void inline fft2(const Mat src, Mat & dest) const; @@ -113,7 +108,7 @@ namespace cv{ bool inline oclTransposeMM(const Mat src, float alpha, UMat &dst); #endif - private: +private: float output_sigma; Rect2d roi; Mat hann; //hann window filter @@ -164,21 +159,14 @@ namespace cv{ #endif int frame; - }; +}; /* * Constructor */ - Ptr TrackerKCF::create(const TrackerKCF::Params ¶meters){ - return Ptr(new TrackerKCFImpl(parameters)); - } - Ptr TrackerKCF::create(){ - return Ptr(new TrackerKCFImpl()); - } TrackerKCFImpl::TrackerKCFImpl( const TrackerKCF::Params ¶meters ) : params( parameters ) { - isInit = false; resizeImage = false; use_custom_extractor_pca = false; use_custom_extractor_npca = false; @@ -195,14 +183,6 @@ namespace cv{ #endif } - void TrackerKCFImpl::read( const cv::FileNode& fn ){ - params.read( fn ); - } - - void TrackerKCFImpl::write( cv::FileStorage& fs ) const { - params.write( fs ); - } - /* * Initialization: * - creating hann window filter @@ -210,7 +190,8 @@ namespace cv{ * - creating a gaussian response for the training ground-truth * - perform FFT to the gaussian response */ - bool TrackerKCFImpl::initImpl( const Mat& image, const Rect2d& boundingBox ){ + void TrackerKCFImpl::init(InputArray image, const Rect& boundingBox) + { frame=0; roi.x = cvRound(boundingBox.x); roi.y = cvRound(boundingBox.y); @@ -262,7 +243,7 @@ namespace cv{ params.desc_pca &= ~(CN); params.desc_npca &= ~(CN); } - model=Ptr(new TrackerKCFModel(params)); + model = makePtr(); // record the non-compressed descriptors if((params.desc_npca & GRAY) == GRAY)descriptors_npca.push_back(GRAY); @@ -286,27 +267,29 @@ namespace cv{ || use_custom_extractor_npca ); - //return true only if roi has intersection with the image - if((roi & Rect2d(0,0, resizeImage ? image.cols / 2 : image.cols, - resizeImage ? image.rows / 2 : image.rows)) == Rect2d()) - return false; - - return true; + // ensure roi has intersection with the image + Rect2d image_roi(0, 0, + image.cols() / (resizeImage ? 2 : 1), + image.rows() / (resizeImage ? 2 : 1)); + CV_Assert(!(roi & image_roi).empty()); } /* * Main part of the KCF algorithm */ - bool TrackerKCFImpl::updateImpl( const Mat& image, Rect2d& boundingBox ){ + bool TrackerKCFImpl::update(InputArray image, Rect& boundingBoxResult) + { double minVal, maxVal; // min-max response Point minLoc,maxLoc; // min-max location - Mat img=image.clone(); - // check the channels of the input image, grayscale is preferred - CV_Assert(img.channels() == 1 || img.channels() == 3); + CV_Assert(image.channels() == 1 || image.channels() == 3); + Mat img; // resize the image whenever needed - if(resizeImage)resize(img,img,Size(img.cols/2,img.rows/2),0,0,INTER_LINEAR_EXACT); + if (resizeImage) + resize(image, img, Size(image.cols()/2, image.rows()/2), 0, 0, INTER_LINEAR_EXACT); + else + image.copyTo(img); // detection part if(frame>0){ @@ -377,6 +360,7 @@ namespace cv{ } // update the bounding box + Rect2d boundingBox; boundingBox.x=(resizeImage?roi.x*2:roi.x)+(resizeImage?roi.width*2:roi.width)/4; boundingBox.y=(resizeImage?roi.y*2:roi.y)+(resizeImage?roi.height*2:roi.height)/4; boundingBox.width = (resizeImage?roi.width*2:roi.width)/2; @@ -475,6 +459,13 @@ namespace cv{ } frame++; + + int x1 = cvRound(boundingBox.x); + int y1 = cvRound(boundingBox.y); + int x2 = cvRound(boundingBox.x + boundingBox.width); + int y2 = cvRound(boundingBox.y + boundingBox.height); + boundingBoxResult = Rect(x1, y1, x2 - x1, y2 - y1) & Rect(Point(0, 0), image.size()); + return true; } @@ -664,7 +655,7 @@ namespace cv{ Rect region=_roi; // return false if roi is outside the image - if((roi & Rect2d(0,0, img.cols, img.rows)) == Rect2d() ) + if ((roi & Rect2d(0, 0, img.cols, img.rows)).empty()) return false; // extract patch inside the image @@ -903,10 +894,11 @@ namespace cv{ } /*----------------------------------------------------------------------*/ - /* - * Parameters - */ - TrackerKCF::Params::Params(){ + +} // namespace + +TrackerKCF::Params::Params() +{ detect_thresh = 0.5f; sigma=0.2f; lambda=0.0001f; @@ -923,69 +915,24 @@ namespace cv{ compress_feature=true; compressed_size=2; pca_learning_rate=0.15f; - } - - void TrackerKCF::Params::read( const cv::FileNode& fn ){ - *this = TrackerKCF::Params(); - - if (!fn["detect_thresh"].empty()) - fn["detect_thresh"] >> detect_thresh; - - if (!fn["sigma"].empty()) - fn["sigma"] >> sigma; +} - if (!fn["lambda"].empty()) - fn["lambda"] >> lambda; - if (!fn["interp_factor"].empty()) - fn["interp_factor"] >> interp_factor; +TrackerKCF::TrackerKCF() +{ + // nothing +} - if (!fn["output_sigma_factor"].empty()) - fn["output_sigma_factor"] >> output_sigma_factor; +TrackerKCF::~TrackerKCF() +{ + // nothing +} - if (!fn["resize"].empty()) - fn["resize"] >> resize; +Ptr TrackerKCF::create(const TrackerKCF::Params ¶meters) +{ + return makePtr(parameters); +} - if (!fn["max_patch_size"].empty()) - fn["max_patch_size"] >> max_patch_size; +}} // namespace - if (!fn["split_coeff"].empty()) - fn["split_coeff"] >> split_coeff; - - if (!fn["wrap_kernel"].empty()) - fn["wrap_kernel"] >> wrap_kernel; - - - if (!fn["desc_npca"].empty()) - fn["desc_npca"] >> desc_npca; - - if (!fn["desc_pca"].empty()) - fn["desc_pca"] >> desc_pca; - - if (!fn["compress_feature"].empty()) - fn["compress_feature"] >> compress_feature; - - if (!fn["compressed_size"].empty()) - fn["compressed_size"] >> compressed_size; - - if (!fn["pca_learning_rate"].empty()) - fn["pca_learning_rate"] >> pca_learning_rate; - } - - void TrackerKCF::Params::write( cv::FileStorage& fs ) const{ - fs << "detect_thresh" << detect_thresh; - fs << "sigma" << sigma; - fs << "lambda" << lambda; - fs << "interp_factor" << interp_factor; - fs << "output_sigma_factor" << output_sigma_factor; - fs << "resize" << resize; - fs << "max_patch_size" << max_patch_size; - fs << "split_coeff" << split_coeff; - fs << "wrap_kernel" << wrap_kernel; - fs << "desc_npca" << desc_npca; - fs << "desc_pca" << desc_pca; - fs << "compress_feature" << compress_feature; - fs << "compressed_size" << compressed_size; - fs << "pca_learning_rate" << pca_learning_rate; - } -} /* namespace cv */ +#include "legacy/trackerKCF.legacy.hpp" diff --git a/modules/tracking/src/trackerMIL.cpp b/modules/tracking/src/trackerMIL.cpp deleted file mode 100644 index 8ccc3b3b348..00000000000 --- a/modules/tracking/src/trackerMIL.cpp +++ /dev/null @@ -1,288 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "precomp.hpp" -#include "trackerMILModel.hpp" - -namespace cv -{ - -class TrackerMILImpl : public TrackerMIL -{ - public: - TrackerMILImpl( const TrackerMIL::Params ¶meters = TrackerMIL::Params() ); - void read( const FileNode& fn ) CV_OVERRIDE; - void write( FileStorage& fs ) const CV_OVERRIDE; - - protected: - - bool initImpl( const Mat& image, const Rect2d& boundingBox ) CV_OVERRIDE; - bool updateImpl( const Mat& image, Rect2d& boundingBox ) CV_OVERRIDE; - void compute_integral( const Mat & img, Mat & ii_img ); - - TrackerMIL::Params params; -}; - -/* - * TrackerMIL - */ - -/* - * Parameters - */ -TrackerMIL::Params::Params() -{ - samplerInitInRadius = 3; - samplerSearchWinSize = 25; - samplerInitMaxNegNum = 65; - samplerTrackInRadius = 4; - samplerTrackMaxPosNum = 100000; - samplerTrackMaxNegNum = 65; - featureSetNumFeatures = 250; -} - -void TrackerMIL::Params::read( const cv::FileNode& fn ) -{ - samplerInitInRadius = fn["samplerInitInRadius"]; - samplerSearchWinSize = fn["samplerSearchWinSize"]; - samplerInitMaxNegNum = fn["samplerInitMaxNegNum"]; - samplerTrackInRadius = fn["samplerTrackInRadius"]; - samplerTrackMaxPosNum = fn["samplerTrackMaxPosNum"]; - samplerTrackMaxNegNum = fn["samplerTrackMaxNegNum"]; - featureSetNumFeatures = fn["featureSetNumFeatures"]; -} - -void TrackerMIL::Params::write( cv::FileStorage& fs ) const -{ - fs << "samplerInitInRadius" << samplerInitInRadius; - fs << "samplerSearchWinSize" << samplerSearchWinSize; - fs << "samplerInitMaxNegNum" << samplerInitMaxNegNum; - fs << "samplerTrackInRadius" << samplerTrackInRadius; - fs << "samplerTrackMaxPosNum" << samplerTrackMaxPosNum; - fs << "samplerTrackMaxNegNum" << samplerTrackMaxNegNum; - fs << "featureSetNumFeatures" << featureSetNumFeatures; - -} - -/* - * Constructor - */ -Ptr TrackerMIL::create(const TrackerMIL::Params ¶meters){ - return Ptr(new TrackerMILImpl(parameters)); -} -Ptr TrackerMIL::create(){ - return Ptr(new TrackerMILImpl()); -} -TrackerMILImpl::TrackerMILImpl( const TrackerMIL::Params ¶meters ) : - params( parameters ) -{ - isInit = false; -} - -void TrackerMILImpl::read( const cv::FileNode& fn ) -{ - params.read( fn ); -} - -void TrackerMILImpl::write( cv::FileStorage& fs ) const -{ - params.write( fs ); -} - -void TrackerMILImpl::compute_integral( const Mat & img, Mat & ii_img ) -{ - Mat ii; - std::vector ii_imgs; - integral( img, ii, CV_32F ); - split( ii, ii_imgs ); - ii_img = ii_imgs[0]; -} - -bool TrackerMILImpl::initImpl( const Mat& image, const Rect2d& boundingBox ) -{ - srand (1); - Mat intImage; - compute_integral( image, intImage ); - TrackerSamplerCSC::Params CSCparameters; - CSCparameters.initInRad = params.samplerInitInRadius; - CSCparameters.searchWinSize = params.samplerSearchWinSize; - CSCparameters.initMaxNegNum = params.samplerInitMaxNegNum; - CSCparameters.trackInPosRad = params.samplerTrackInRadius; - CSCparameters.trackMaxPosNum = params.samplerTrackMaxPosNum; - CSCparameters.trackMaxNegNum = params.samplerTrackMaxNegNum; - - Ptr CSCSampler = Ptr( new TrackerSamplerCSC( CSCparameters ) ); - if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) - return false; - - //or add CSC sampler with default parameters - //sampler->addTrackerSamplerAlgorithm( "CSC" ); - - //Positive sampling - CSCSampler.staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_POS ); - sampler->sampling( intImage, boundingBox ); - std::vector posSamples = sampler->getSamples(); - - //Negative sampling - CSCSampler.staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_NEG ); - sampler->sampling( intImage, boundingBox ); - std::vector negSamples = sampler->getSamples(); - - if( posSamples.empty() || negSamples.empty() ) - return false; - - //compute HAAR features - TrackerFeatureHAAR::Params HAARparameters; - HAARparameters.numFeatures = params.featureSetNumFeatures; - HAARparameters.rectSize = Size( (int)boundingBox.width, (int)boundingBox.height ); - HAARparameters.isIntegral = true; - Ptr trackerFeature = Ptr( new TrackerFeatureHAAR( HAARparameters ) ); - featureSet->addTrackerFeature( trackerFeature ); - - featureSet->extraction( posSamples ); - const std::vector posResponse = featureSet->getResponses(); - - featureSet->extraction( negSamples ); - const std::vector negResponse = featureSet->getResponses(); - - model = Ptr( new TrackerMILModel( boundingBox ) ); - Ptr stateEstimator = Ptr( - new TrackerStateEstimatorMILBoosting( params.featureSetNumFeatures ) ); - model->setTrackerStateEstimator( stateEstimator ); - - //Run model estimation and update - model.staticCast()->setMode( TrackerMILModel::MODE_POSITIVE, posSamples ); - model->modelEstimation( posResponse ); - model.staticCast()->setMode( TrackerMILModel::MODE_NEGATIVE, negSamples ); - model->modelEstimation( negResponse ); - model->modelUpdate(); - - return true; -} - -bool TrackerMILImpl::updateImpl( const Mat& image, Rect2d& boundingBox ) -{ - Mat intImage; - compute_integral( image, intImage ); - - //get the last location [AAM] X(k-1) - Ptr lastLocation = model->getLastTargetState(); - Rect lastBoundingBox( (int)lastLocation->getTargetPosition().x, (int)lastLocation->getTargetPosition().y, lastLocation->getTargetWidth(), - lastLocation->getTargetHeight() ); - - //sampling new frame based on last location - ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_DETECT ); - sampler->sampling( intImage, lastBoundingBox ); - std::vector detectSamples = sampler->getSamples(); - if( detectSamples.empty() ) - return false; - - /*//TODO debug samples - Mat f; - image.copyTo(f); - - for( size_t i = 0; i < detectSamples.size(); i=i+10 ) - { - Size sz; - Point off; - detectSamples.at(i).locateROI(sz, off); - rectangle(f, Rect(off.x,off.y,detectSamples.at(i).cols,detectSamples.at(i).rows), Scalar(255,0,0), 1); - }*/ - - //extract features from new samples - featureSet->extraction( detectSamples ); - std::vector response = featureSet->getResponses(); - - //predict new location - ConfidenceMap cmap; - model.staticCast()->setMode( TrackerMILModel::MODE_ESTIMATON, detectSamples ); - model.staticCast()->responseToConfidenceMap( response, cmap ); - model->getTrackerStateEstimator().staticCast()->setCurrentConfidenceMap( cmap ); - - if( !model->runStateEstimator() ) - { - return false; - } - - Ptr currentState = model->getLastTargetState(); - boundingBox = Rect( (int)currentState->getTargetPosition().x, (int)currentState->getTargetPosition().y, currentState->getTargetWidth(), - currentState->getTargetHeight() ); - - /*//TODO debug - rectangle(f, lastBoundingBox, Scalar(0,255,0), 1); - rectangle(f, boundingBox, Scalar(0,0,255), 1); - imshow("f", f); - //waitKey( 0 );*/ - - //sampling new frame based on new location - //Positive sampling - ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_POS ); - sampler->sampling( intImage, boundingBox ); - std::vector posSamples = sampler->getSamples(); - - //Negative sampling - ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_NEG ); - sampler->sampling( intImage, boundingBox ); - std::vector negSamples = sampler->getSamples(); - - if( posSamples.empty() || negSamples.empty() ) - return false; - - //extract features - featureSet->extraction( posSamples ); - std::vector posResponse = featureSet->getResponses(); - - featureSet->extraction( negSamples ); - std::vector negResponse = featureSet->getResponses(); - - //model estimate - model.staticCast()->setMode( TrackerMILModel::MODE_POSITIVE, posSamples ); - model->modelEstimation( posResponse ); - model.staticCast()->setMode( TrackerMILModel::MODE_NEGATIVE, negSamples ); - model->modelEstimation( negResponse ); - - //model update - model->modelUpdate(); - - return true; -} - -} /* namespace cv */ diff --git a/modules/tracking/src/trackerMILModel.cpp b/modules/tracking/src/trackerMILModel.cpp deleted file mode 100644 index f6abec68f83..00000000000 --- a/modules/tracking/src/trackerMILModel.cpp +++ /dev/null @@ -1,125 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "precomp.hpp" -#include "trackerMILModel.hpp" - -/** - * TrackerMILModel - */ - -namespace cv -{ - -TrackerMILModel::TrackerMILModel( const Rect& boundingBox ) -{ - currentSample.clear(); - mode = MODE_POSITIVE; - width = boundingBox.width; - height = boundingBox.height; - - Ptr initState = Ptr( - new TrackerStateEstimatorMILBoosting::TrackerMILTargetState( Point2f( (float)boundingBox.x, (float)boundingBox.y ), boundingBox.width, boundingBox.height, - true, Mat() ) ); - trajectory.push_back( initState ); -} - -void TrackerMILModel::responseToConfidenceMap( const std::vector& responses, ConfidenceMap& confidenceMap ) -{ - if( currentSample.empty() ) - { - CV_Error( -1, "The samples in Model estimation are empty" ); - } - - for ( size_t i = 0; i < responses.size(); i++ ) - { - //for each column (one sample) there are #num_feature - //get informations from currentSample - for ( int j = 0; j < responses.at( i ).cols; j++ ) - { - - Size currentSize; - Point currentOfs; - currentSample.at( j ).locateROI( currentSize, currentOfs ); - bool foreground = false; - if( mode == MODE_POSITIVE || mode == MODE_ESTIMATON ) - { - foreground = true; - } - else if( mode == MODE_NEGATIVE ) - { - foreground = false; - } - - //get the column of the HAAR responses - Mat singleResponse = responses.at( i ).col( j ); - - //create the state - Ptr currentState = Ptr( - new TrackerStateEstimatorMILBoosting::TrackerMILTargetState( currentOfs, width, height, foreground, singleResponse ) ); - - confidenceMap.push_back( std::make_pair( currentState, 0.0f ) ); - - } - - } -} - -void TrackerMILModel::modelEstimationImpl( const std::vector& responses ) -{ - responseToConfidenceMap( responses, currentConfidenceMap ); - -} - -void TrackerMILModel::modelUpdateImpl() -{ - -} - -void TrackerMILModel::setMode( int trainingMode, const std::vector& samples ) -{ - currentSample.clear(); - currentSample = samples; - - mode = trainingMode; -} - -} diff --git a/modules/tracking/src/trackerMILModel.hpp b/modules/tracking/src/trackerMILModel.hpp deleted file mode 100644 index cdbf9a3cb1c..00000000000 --- a/modules/tracking/src/trackerMILModel.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#ifndef __OPENCV_TRACKER_MIL_MODEL_HPP__ -#define __OPENCV_TRACKER_MIL_MODEL_HPP__ - -#include "opencv2/core.hpp" - -namespace cv -{ - -/** - * \brief Implementation of TrackerModel for MIL algorithm - */ -class TrackerMILModel : public TrackerModel -{ - public: - enum - { - MODE_POSITIVE = 1, // mode for positive features - MODE_NEGATIVE = 2, // mode for negative features - MODE_ESTIMATON = 3 // mode for estimation step - }; - - /** - * \brief Constructor - * \param boundingBox The first boundingBox - */ - TrackerMILModel( const Rect& boundingBox ); - - /** - * \brief Destructor - */ - ~TrackerMILModel() - { - } - ; - - /** - * \brief Set the mode - */ - void setMode( int trainingMode, const std::vector& samples ); - - /** - * \brief Create the ConfidenceMap from a list of responses - * \param responses The list of the responses - * \param confidenceMap The output - */ - void responseToConfidenceMap( const std::vector& responses, ConfidenceMap& confidenceMap ); - - protected: - void modelEstimationImpl( const std::vector& responses ) CV_OVERRIDE; - void modelUpdateImpl() CV_OVERRIDE; - - private: - int mode; - std::vector currentSample; - - int width; //initial width of the boundingBox - int height; //initial height of the boundingBox -}; - -} /* namespace cv */ - -#endif diff --git a/modules/tracking/src/trackerMIL_legacy.cpp b/modules/tracking/src/trackerMIL_legacy.cpp new file mode 100644 index 00000000000..d6dc3cd2cdc --- /dev/null +++ b/modules/tracking/src/trackerMIL_legacy.cpp @@ -0,0 +1,128 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +namespace legacy { +inline namespace tracking { +namespace impl { + +class TrackerMILImpl CV_FINAL : public legacy::TrackerMIL +{ +public: + Ptr impl; + legacy::TrackerMIL::Params params; + + TrackerMILImpl(const legacy::TrackerMIL::Params ¶meters) + : impl(cv::TrackerMIL::create(parameters)) + , params(parameters) + { + isInit = false; + } + + void read(const FileNode& fn) CV_OVERRIDE + { + params.read(fn); + CV_Error(Error::StsNotImplemented, "Can't update legacy tracker wrapper"); + } + void write(FileStorage& fs) const CV_OVERRIDE + { + params.write(fs); + } + + bool initImpl(const Mat& image, const Rect2d& boundingBox2d) CV_OVERRIDE + { + int x1 = cvRound(boundingBox2d.x); + int y1 = cvRound(boundingBox2d.y); + int x2 = cvRound(boundingBox2d.x + boundingBox2d.width); + int y2 = cvRound(boundingBox2d.y + boundingBox2d.height); + Rect boundingBox = Rect(x1, y1, x2 - x1, y2 - y1) & Rect(Point(0, 0), image.size()); + impl->init(image, boundingBox); + isInit = true; + return true; + } + bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE + { + Rect bb; + bool res = impl->update(image, bb); + boundingBox = bb; + return res; + } +}; + +} // namespace + +void legacy::TrackerMIL::Params::read(const cv::FileNode& fn) +{ + samplerInitInRadius = fn["samplerInitInRadius"]; + samplerSearchWinSize = fn["samplerSearchWinSize"]; + samplerInitMaxNegNum = fn["samplerInitMaxNegNum"]; + samplerTrackInRadius = fn["samplerTrackInRadius"]; + samplerTrackMaxPosNum = fn["samplerTrackMaxPosNum"]; + samplerTrackMaxNegNum = fn["samplerTrackMaxNegNum"]; + featureSetNumFeatures = fn["featureSetNumFeatures"]; +} + +void legacy::TrackerMIL::Params::write(cv::FileStorage& fs) const +{ + fs << "samplerInitInRadius" << samplerInitInRadius; + fs << "samplerSearchWinSize" << samplerSearchWinSize; + fs << "samplerInitMaxNegNum" << samplerInitMaxNegNum; + fs << "samplerTrackInRadius" << samplerTrackInRadius; + fs << "samplerTrackMaxPosNum" << samplerTrackMaxPosNum; + fs << "samplerTrackMaxNegNum" << samplerTrackMaxNegNum; + fs << "featureSetNumFeatures" << featureSetNumFeatures; +} + +}} // namespace + +Ptr legacy::TrackerMIL::create(const legacy::TrackerMIL::Params ¶meters) +{ + return makePtr(parameters); +} +Ptr legacy::TrackerMIL::create() +{ + return create(legacy::TrackerMIL::Params()); +} + +} // namespace diff --git a/modules/tracking/src/trackerMedianFlow.cpp b/modules/tracking/src/trackerMedianFlow.cpp index 6f41c3ebe6c..f183448e231 100644 --- a/modules/tracking/src/trackerMedianFlow.cpp +++ b/modules/tracking/src/trackerMedianFlow.cpp @@ -40,15 +40,15 @@ //M*/ #include "precomp.hpp" -#include "opencv2/video/tracking.hpp" -#include "opencv2/imgproc.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" + #include "tracking_utils.hpp" #include #include -namespace -{ -using namespace cv; +namespace cv { +inline namespace tracking { +namespace impl { #undef MEDIAN_FLOW_TRACKER_DEBUG_LOGS #ifdef MEDIAN_FLOW_TRACKER_DEBUG_LOGS @@ -72,7 +72,8 @@ using namespace cv; * optimize (allocation<-->reallocation) */ -class TrackerMedianFlowImpl : public TrackerMedianFlow{ +class TrackerMedianFlowImpl : public legacy::TrackerMedianFlow +{ public: TrackerMedianFlowImpl(TrackerMedianFlow::Params paramsIn = TrackerMedianFlow::Params()) {params=paramsIn;isInit=false;} void read( const FileNode& fn ) CV_OVERRIDE; @@ -95,6 +96,7 @@ class TrackerMedianFlowImpl : public TrackerMedianFlow{ TrackerMedianFlow::Params params; }; +static Mat getPatch(Mat image, Size patch_size, Point2f patch_center) { Mat patch; @@ -119,7 +121,7 @@ Mat getPatch(Mat image, Size patch_size, Point2f patch_center) class TrackerMedianFlowModel : public TrackerModel{ public: - TrackerMedianFlowModel(TrackerMedianFlow::Params /*params*/){} + TrackerMedianFlowModel(legacy::TrackerMedianFlow::Params /*params*/){} Rect2d getBoundingBox(){return boundingBox_;} void setBoudingBox(Rect2d boundingBox){boundingBox_=boundingBox;} Mat getImage(){return image_;} @@ -391,10 +393,11 @@ void TrackerMedianFlowImpl::check_NCC(const Mat& oldImage,const Mat& newImage, } } -} /* anonymous namespace */ +}} // namespace + +namespace legacy { +inline namespace tracking { -namespace cv -{ /* * Parameters */ @@ -442,11 +445,13 @@ void TrackerMedianFlow::Params::write( cv::FileStorage& fs ) const{ fs << "maxMedianLengthOfDisplacementDifference" << maxMedianLengthOfDisplacementDifference; } -Ptr TrackerMedianFlow::create(const TrackerMedianFlow::Params ¶meters){ - return Ptr(new TrackerMedianFlowImpl(parameters)); +Ptr TrackerMedianFlow::create(const TrackerMedianFlow::Params ¶meters) +{ + return makePtr(parameters); } -Ptr TrackerMedianFlow::create(){ - return Ptr(new TrackerMedianFlowImpl()); +Ptr TrackerMedianFlow::create() +{ + return create(TrackerMedianFlow::Params()); } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/trackerModel.cpp b/modules/tracking/src/trackerModel.cpp deleted file mode 100644 index 023b3b67dc3..00000000000 --- a/modules/tracking/src/trackerModel.cpp +++ /dev/null @@ -1,177 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "precomp.hpp" - -namespace cv -{ - -/* - * TrackerModel - */ - -TrackerModel::TrackerModel() -{ - stateEstimator = Ptr(); - maxCMLength = 10; -} - -TrackerModel::~TrackerModel() -{ - -} - -bool TrackerModel::setTrackerStateEstimator( Ptr trackerStateEstimator ) -{ - if (stateEstimator.get()) - { - return false; - } - - stateEstimator = trackerStateEstimator; - return true; -} - -Ptr TrackerModel::getTrackerStateEstimator() const -{ - return stateEstimator; -} - -void TrackerModel::modelEstimation( const std::vector& responses ) -{ - modelEstimationImpl( responses ); - -} - -void TrackerModel::clearCurrentConfidenceMap() -{ - currentConfidenceMap.clear(); -} - -void TrackerModel::modelUpdate() -{ - modelUpdateImpl(); - - if( maxCMLength != -1 && (int) confidenceMaps.size() >= maxCMLength - 1 ) - { - int l = maxCMLength / 2; - confidenceMaps.erase( confidenceMaps.begin(), confidenceMaps.begin() + l ); - } - if( maxCMLength != -1 && (int) trajectory.size() >= maxCMLength - 1 ) - { - int l = maxCMLength / 2; - trajectory.erase( trajectory.begin(), trajectory.begin() + l ); - } - confidenceMaps.push_back( currentConfidenceMap ); - stateEstimator->update( confidenceMaps ); - - clearCurrentConfidenceMap(); - -} - -bool TrackerModel::runStateEstimator() -{ - if (!stateEstimator) - { - CV_Error( -1, "Tracker state estimator is not setted" ); - } - Ptr targetState = stateEstimator->estimate( confidenceMaps ); - if (!targetState) - return false; - - setLastTargetState( targetState ); - return true; -} - -void TrackerModel::setLastTargetState( const Ptr& lastTargetState ) -{ - trajectory.push_back( lastTargetState ); -} - -Ptr TrackerModel::getLastTargetState() const -{ - return trajectory.back(); -} - -const std::vector& TrackerModel::getConfidenceMaps() const -{ - return confidenceMaps; -} - -const ConfidenceMap& TrackerModel::getLastConfidenceMap() const -{ - return confidenceMaps.back(); -} - -/* - * TrackerTargetState - */ - -Point2f TrackerTargetState::getTargetPosition() const -{ - return targetPosition; -} - -void TrackerTargetState::setTargetPosition( const Point2f& position ) -{ - targetPosition = position; -} - -int TrackerTargetState::getTargetWidth() const -{ - return targetWidth; -} - -void TrackerTargetState::setTargetWidth( int width ) -{ - targetWidth = width; -} -int TrackerTargetState::getTargetHeight() const -{ - return targetHeight; -} - -void TrackerTargetState::setTargetHeight( int height ) -{ - targetHeight = height; -} - -} /* namespace cv */ diff --git a/modules/tracking/src/trackerSampler.cpp b/modules/tracking/src/trackerSampler.cpp index 38ea52317e3..f4da8ba058c 100644 --- a/modules/tracking/src/trackerSampler.cpp +++ b/modules/tracking/src/trackerSampler.cpp @@ -41,17 +41,18 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* - * TrackerSampler + * TrackerContribSampler */ /* * Constructor */ -TrackerSampler::TrackerSampler() +TrackerContribSampler::TrackerContribSampler() { blockAddTrackerSampler = false; } @@ -59,12 +60,12 @@ TrackerSampler::TrackerSampler() /* * Destructor */ -TrackerSampler::~TrackerSampler() +TrackerContribSampler::~TrackerContribSampler() { } -void TrackerSampler::sampling( const Mat& image, Rect boundingBox ) +void TrackerContribSampler::sampling( const Mat& image, Rect boundingBox ) { clearSamples(); @@ -88,13 +89,13 @@ void TrackerSampler::sampling( const Mat& image, Rect boundingBox ) } } -bool TrackerSampler::addTrackerSamplerAlgorithm( String trackerSamplerAlgorithmType ) +bool TrackerContribSampler::addTrackerSamplerAlgorithm( String trackerSamplerAlgorithmType ) { if( blockAddTrackerSampler ) { return false; } - Ptr sampler = TrackerSamplerAlgorithm::create( trackerSamplerAlgorithmType ); + Ptr sampler = TrackerContribSamplerAlgorithm::create( trackerSamplerAlgorithmType ); if (!sampler) { @@ -106,7 +107,7 @@ bool TrackerSampler::addTrackerSamplerAlgorithm( String trackerSamplerAlgorithmT return true; } -bool TrackerSampler::addTrackerSamplerAlgorithm( Ptr& sampler ) +bool TrackerContribSampler::addTrackerSamplerAlgorithm( Ptr& sampler ) { if( blockAddTrackerSampler ) { @@ -124,19 +125,20 @@ bool TrackerSampler::addTrackerSamplerAlgorithm( Ptr& s return true; } -const std::vector > >& TrackerSampler::getSamplers() const +const std::vector > >& TrackerContribSampler::getSamplers() const { return samplers; } -const std::vector& TrackerSampler::getSamples() const +const std::vector& TrackerContribSampler::getSamples() const { return samples; } -void TrackerSampler::clearSamples() +void TrackerContribSampler::clearSamples() { samples.clear(); } -} /* namespace cv */ + +}}} // namespace diff --git a/modules/tracking/src/trackerSamplerAlgorithm.cpp b/modules/tracking/src/trackerSamplerAlgorithm.cpp index ea52070d59c..75dd9a296a2 100644 --- a/modules/tracking/src/trackerSamplerAlgorithm.cpp +++ b/modules/tracking/src/trackerSamplerAlgorithm.cpp @@ -40,29 +40,23 @@ //M*/ #include "precomp.hpp" -#include #include "PFSolver.hpp" #include "TrackingFunctionPF.hpp" -#ifdef _WIN32 -#define TIME( arg ) (((double) clock()) / CLOCKS_PER_SEC) -#else -#define TIME( arg ) (time( arg )) -#endif - -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* - * TrackerSamplerAlgorithm + * TrackerContribSamplerAlgorithm */ -TrackerSamplerAlgorithm::~TrackerSamplerAlgorithm() +TrackerContribSamplerAlgorithm::~TrackerContribSamplerAlgorithm() { } -bool TrackerSamplerAlgorithm::sampling( const Mat& image, Rect boundingBox, std::vector& sample ) +bool TrackerContribSamplerAlgorithm::sampling(const Mat& image, const Rect& boundingBox, std::vector& sample) { if( image.empty() ) return false; @@ -70,11 +64,11 @@ bool TrackerSamplerAlgorithm::sampling( const Mat& image, Rect boundingBox, std: return samplingImpl( image, boundingBox, sample ); } -Ptr TrackerSamplerAlgorithm::create( const String& trackerSamplerType ) +Ptr TrackerContribSamplerAlgorithm::create( const String& trackerSamplerType ) { if( trackerSamplerType.find( "CSC" ) == 0 ) { - return Ptr( new TrackerSamplerCSC() ); + return Ptr( new TrackerContribSamplerCSC() ); } if( trackerSamplerType.find( "CS" ) == 0 ) @@ -82,23 +76,23 @@ Ptr TrackerSamplerAlgorithm::create( const String& trac return Ptr( new TrackerSamplerCS() ); } - CV_Error( -1, "Tracker sampler algorithm type not supported" ); + CV_Error(Error::StsNotImplemented, "Tracker sampler algorithm type not supported" ); } -String TrackerSamplerAlgorithm::getClassName() const +String TrackerContribSamplerAlgorithm::getClassName() const { return className; } /** - * TrackerSamplerCSC + * TrackerContribSamplerCSC */ /** * Parameters */ -TrackerSamplerCSC::Params::Params() +TrackerContribSamplerCSC::Params::Params() { initInRad = 3; initMaxNegNum = 65; @@ -109,21 +103,21 @@ TrackerSamplerCSC::Params::Params() } -TrackerSamplerCSC::TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meters ) : +TrackerContribSamplerCSC::TrackerContribSamplerCSC( const TrackerContribSamplerCSC::Params ¶meters ) : params( parameters ) { className = "CSC"; mode = MODE_INIT_POS; - rng = RNG( uint64( TIME( 0 ) ) ); + rng = theRNG(); } -TrackerSamplerCSC::~TrackerSamplerCSC() +TrackerContribSamplerCSC::~TrackerContribSamplerCSC() { } -bool TrackerSamplerCSC::samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) +bool TrackerContribSamplerCSC::samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) { float inrad = 0; float outrad = 0; @@ -165,12 +159,12 @@ bool TrackerSamplerCSC::samplingImpl( const Mat& image, Rect boundingBox, std::v return false; } -void TrackerSamplerCSC::setMode( int samplingMode ) +void TrackerContribSamplerCSC::setMode( int samplingMode ) { mode = samplingMode; } -std::vector TrackerSamplerCSC::sampleImage( const Mat& img, int x, int y, int w, int h, float inrad, float outrad, int maxnum ) +std::vector TrackerContribSamplerCSC::sampleImage( const Mat& img, int x, int y, int w, int h, float inrad, float outrad, int maxnum ) { int rowsz = img.rows - h - 1; int colsz = img.cols - w - 1; @@ -410,4 +404,5 @@ bool TrackerSamplerPF::samplingImpl( const Mat& image, Rect boundingBox, std::ve return true; } -} /* namespace cv */ + +}}} // namespace diff --git a/modules/tracking/src/trackerStateEstimator.cpp b/modules/tracking/src/trackerStateEstimator.cpp index 094aef059e4..46b8a815b3b 100644 --- a/modules/tracking/src/trackerStateEstimator.cpp +++ b/modules/tracking/src/trackerStateEstimator.cpp @@ -41,35 +41,9 @@ #include "precomp.hpp" -namespace cv -{ - -/* - * TrackerStateEstimator - */ - -TrackerStateEstimator::~TrackerStateEstimator() -{ - -} - -Ptr TrackerStateEstimator::estimate( const std::vector& confidenceMaps ) -{ - if( confidenceMaps.empty() ) - return Ptr(); - - return estimateImpl( confidenceMaps ); - -} - -void TrackerStateEstimator::update( std::vector& confidenceMaps ) -{ - if( confidenceMaps.empty() ) - return; - - return updateImpl( confidenceMaps ); - -} +namespace cv { +namespace detail { +inline namespace tracking { Ptr TrackerStateEstimator::create( const String& trackeStateEstimatorType ) { @@ -81,164 +55,14 @@ Ptr TrackerStateEstimator::create( const String& trackeSt if( trackeStateEstimatorType.find( "BOOSTING" ) == 0 ) { - return Ptr( new TrackerStateEstimatorMILBoosting() ); + CV_Error(Error::StsNotImplemented, "TrackerStateEstimatorMILBoosting API is not available"); + //return Ptr( new TrackerStateEstimatorMILBoosting() ); } CV_Error( -1, "Tracker state estimator type not supported" ); } -String TrackerStateEstimator::getClassName() const -{ - return className; -} - -/** - * TrackerStateEstimatorMILBoosting::TrackerMILTargetState - */ -TrackerStateEstimatorMILBoosting::TrackerMILTargetState::TrackerMILTargetState( const Point2f& position, int width, int height, bool foreground, - const Mat& features ) -{ - setTargetPosition( position ); - setTargetWidth( width ); - setTargetHeight( height ); - setTargetFg( foreground ); - setFeatures( features ); -} - -void TrackerStateEstimatorMILBoosting::TrackerMILTargetState::setTargetFg( bool foreground ) -{ - isTarget = foreground; -} - -void TrackerStateEstimatorMILBoosting::TrackerMILTargetState::setFeatures( const Mat& features ) -{ - targetFeatures = features; -} - -bool TrackerStateEstimatorMILBoosting::TrackerMILTargetState::isTargetFg() const -{ - return isTarget; -} - -Mat TrackerStateEstimatorMILBoosting::TrackerMILTargetState::getFeatures() const -{ - return targetFeatures; -} - -TrackerStateEstimatorMILBoosting::TrackerStateEstimatorMILBoosting( int nFeatures ) -{ - className = "BOOSTING"; - trained = false; - numFeatures = nFeatures; -} - -TrackerStateEstimatorMILBoosting::~TrackerStateEstimatorMILBoosting() -{ - -} - -void TrackerStateEstimatorMILBoosting::setCurrentConfidenceMap( ConfidenceMap& confidenceMap ) -{ - currentConfidenceMap.clear(); - currentConfidenceMap = confidenceMap; -} - -uint TrackerStateEstimatorMILBoosting::max_idx( const std::vector &v ) -{ - const float* findPtr = & ( *std::max_element( v.begin(), v.end() ) ); - const float* beginPtr = & ( *v.begin() ); - return (uint) ( findPtr - beginPtr ); -} - -Ptr TrackerStateEstimatorMILBoosting::estimateImpl( const std::vector& /*confidenceMaps*/) -{ - //run ClfMilBoost classify in order to compute next location - if( currentConfidenceMap.empty() ) - return Ptr(); - - Mat positiveStates; - Mat negativeStates; - - prepareData( currentConfidenceMap, positiveStates, negativeStates ); - - std::vector prob = boostMILModel.classify( positiveStates ); - - int bestind = max_idx( prob ); - //float resp = prob[bestind]; - - return currentConfidenceMap.at( bestind ).first; -} - -void TrackerStateEstimatorMILBoosting::prepareData( const ConfidenceMap& confidenceMap, Mat& positive, Mat& negative ) -{ - - int posCounter = 0; - int negCounter = 0; - - for ( size_t i = 0; i < confidenceMap.size(); i++ ) - { - Ptr currentTargetState = confidenceMap.at( i ).first.staticCast(); - if( currentTargetState->isTargetFg() ) - posCounter++; - else - negCounter++; - } - positive.create( posCounter, numFeatures, CV_32FC1 ); - negative.create( negCounter, numFeatures, CV_32FC1 ); - - //TODO change with mat fast access - //initialize trainData (positive and negative) - - int pc = 0; - int nc = 0; - for ( size_t i = 0; i < confidenceMap.size(); i++ ) - { - Ptr currentTargetState = confidenceMap.at( i ).first.staticCast(); - Mat stateFeatures = currentTargetState->getFeatures(); - - if( currentTargetState->isTargetFg() ) - { - for ( int j = 0; j < stateFeatures.rows; j++ ) - { - //fill the positive trainData with the value of the feature j for sample i - positive.at( pc, j ) = stateFeatures.at( j, 0 ); - } - pc++; - } - else - { - for ( int j = 0; j < stateFeatures.rows; j++ ) - { - //fill the negative trainData with the value of the feature j for sample i - negative.at( nc, j ) = stateFeatures.at( j, 0 ); - } - nc++; - } - - } -} - -void TrackerStateEstimatorMILBoosting::updateImpl( std::vector& confidenceMaps ) -{ - - if( !trained ) - { - //this is the first time that the classifier is built - //init MIL - boostMILModel.init(); - trained = true; - } - - ConfidenceMap lastConfidenceMap = confidenceMaps.back(); - Mat positiveStates; - Mat negativeStates; - - prepareData( lastConfidenceMap, positiveStates, negativeStates ); - //update MIL - boostMILModel.update( positiveStates, negativeStates ); - -} /** * TrackerStateEstimatorAdaBoosting @@ -441,4 +265,4 @@ void TrackerStateEstimatorSVM::updateImpl( std::vector& /*confide } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/tracking_by_matching.cpp b/modules/tracking/src/tracking_by_matching.cpp index 3e2199b5c7a..8b0228306d5 100644 --- a/modules/tracking/src/tracking_by_matching.cpp +++ b/modules/tracking/src/tracking_by_matching.cpp @@ -2,6 +2,8 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +#include "precomp.hpp" + #include #include #include @@ -25,7 +27,11 @@ #define TBM_CHECK_LE(actual, expected) CV_CheckLE(actual, expected, "Assertion error:") #define TBM_CHECK_GE(actual, expected) CV_CheckGE(actual, expected, "Assertion error:") -using namespace cv::tbm; +namespace cv { +namespace detail { +inline namespace tracking { + +using namespace tbm; CosDistance::CosDistance(const cv::Size &descriptor_size) : descriptor_size_(descriptor_size) { @@ -1335,3 +1341,5 @@ void TrackerByMatching::PrintConfusionMatrices() const { std::cout << cm << std::endl << std::endl; } } + +}}} // namespace diff --git a/modules/tracking/src/tracking_utils.cpp b/modules/tracking/src/tracking_utils.cpp index fd6ae1bc5f1..c197f902d6d 100644 --- a/modules/tracking/src/tracking_utils.cpp +++ b/modules/tracking/src/tracking_utils.cpp @@ -2,9 +2,10 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +#include "precomp.hpp" #include "tracking_utils.hpp" -using namespace cv; +namespace cv { double tracking_internal::computeNCC(const Mat& patch1, const Mat& patch2) { @@ -67,3 +68,5 @@ double tracking_internal::computeNCC(const Mat& patch1, const Mat& patch2) return (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; } } + +} // namespace diff --git a/modules/tracking/src/tracking_utils.hpp b/modules/tracking/src/tracking_utils.hpp index a7356224274..903b783b828 100644 --- a/modules/tracking/src/tracking_utils.hpp +++ b/modules/tracking/src/tracking_utils.hpp @@ -3,12 +3,11 @@ // of this distribution and at http://opencv.org/license.html. #ifndef __OPENCV_TRACKING_UTILS_HPP__ -#include "precomp.hpp" #include namespace cv { -namespace tracking_internal -{ +namespace tracking_internal { + /** Computes normalized corellation coefficient between the two patches (they should be * of the same size).*/ double computeNCC(const Mat& patch1, const Mat& patch2); @@ -42,6 +41,6 @@ namespace tracking_internal std::vector copy(values); return getMedianAndDoPartition(copy); } -} -} + +}} // namespace #endif diff --git a/modules/tracking/src/unscented_kalman.cpp b/modules/tracking/src/unscented_kalman.cpp index 9b800160465..d6d82a7f3bf 100644 --- a/modules/tracking/src/unscented_kalman.cpp +++ b/modules/tracking/src/unscented_kalman.cpp @@ -42,10 +42,10 @@ #include "precomp.hpp" #include "opencv2/tracking/kalman_filters.hpp" -namespace cv -{ -namespace tracking -{ +namespace cv { +namespace detail { +inline namespace tracking { +inline namespace kalman_filters { void UnscentedKalmanFilterParams:: init( int dp, int mp, int cp, double processNoiseCovDiag, double measurementNoiseCovDiag, @@ -369,5 +369,4 @@ Ptr createUnscentedKalmanFilter(const UnscentedKalmanFilt return kfu; } -} // tracking -} // cv +}}}} // namespace diff --git a/modules/tracking/test/test_aukf.cpp b/modules/tracking/test/test_aukf.cpp index 73372971c06..32e32c96e6f 100644 --- a/modules/tracking/test/test_aukf.cpp +++ b/modules/tracking/test/test_aukf.cpp @@ -43,7 +43,7 @@ #include "opencv2/tracking/kalman_filters.hpp" namespace opencv_test { namespace { -using namespace cv::tracking; +using namespace cv::detail; // In this two tests Augmented Unscented Kalman Filter are applied to the dynamic system from example "The reentry problem" from // "A New Extension of the Kalman Filter to Nonlinear Systems" by Simon J. Julier and Jeffrey K. Uhlmann. diff --git a/modules/tracking/test/test_trackerParametersIO.cpp b/modules/tracking/test/test_trackerParametersIO.cpp index ebe43a5fb7d..59e7917e271 100644 --- a/modules/tracking/test/test_trackerParametersIO.cpp +++ b/modules/tracking/test/test_trackerParametersIO.cpp @@ -4,11 +4,15 @@ #include "test_precomp.hpp" +#include +//using namespace cv::tracking::legacy; + namespace opencv_test { namespace { + TEST(MEDIAN_FLOW_Parameters, IO) { - TrackerMedianFlow::Params parameters; + legacy::TrackerMedianFlow::Params parameters; parameters.maxLevel = 10; parameters.maxMedianLengthOfDisplacementDifference = 11; @@ -25,7 +29,7 @@ TEST(MEDIAN_FLOW_Parameters, IO) FileStorage fsReader(serializedParameters, FileStorage::READ + FileStorage::MEMORY); - TrackerMedianFlow::Params readParameters; + legacy::TrackerMedianFlow::Params readParameters; readParameters.read(fsReader.root()); ASSERT_EQ(parameters.maxLevel, readParameters.maxLevel); @@ -41,11 +45,11 @@ TEST(MEDIAN_FLOW_Parameters, IO) TEST(MEDIAN_FLOW_Parameters, Default_Value_If_Absent) { - TrackerMedianFlow::Params defaultParameters; + legacy::TrackerMedianFlow::Params defaultParameters; FileStorage fsReader(String("%YAML 1.0"), FileStorage::READ + FileStorage::MEMORY); - TrackerMedianFlow::Params readParameters; + legacy::TrackerMedianFlow::Params readParameters; readParameters.read(fsReader.root()); ASSERT_EQ(defaultParameters.maxLevel, readParameters.maxLevel); @@ -60,7 +64,7 @@ TEST(MEDIAN_FLOW_Parameters, Default_Value_If_Absent) TEST(KCF_Parameters, IO) { - TrackerKCF::Params parameters; + legacy::TrackerKCF::Params parameters; parameters.sigma = 0.3f; parameters.lambda = 0.02f; @@ -83,7 +87,7 @@ TEST(KCF_Parameters, IO) FileStorage fsReader(serializedParameters, FileStorage::READ + FileStorage::MEMORY); - TrackerKCF::Params readParameters; + legacy::TrackerKCF::Params readParameters; readParameters.read(fsReader.root()); ASSERT_DOUBLE_EQ(parameters.sigma, readParameters.sigma); @@ -103,11 +107,11 @@ TEST(KCF_Parameters, IO) TEST(KCF_Parameters, Default_Value_If_Absent) { - TrackerKCF::Params defaultParameters; + legacy::TrackerKCF::Params defaultParameters; FileStorage fsReader(String("%YAML 1.0"), FileStorage::READ + FileStorage::MEMORY); - TrackerKCF::Params readParameters; + legacy::TrackerKCF::Params readParameters; readParameters.read(fsReader.root()); ASSERT_DOUBLE_EQ(defaultParameters.sigma, readParameters.sigma); diff --git a/modules/tracking/test/test_trackers.cpp b/modules/tracking/test/test_trackers.cpp index db1617e3500..1241bbba79b 100644 --- a/modules/tracking/test/test_trackers.cpp +++ b/modules/tracking/test/test_trackers.cpp @@ -41,7 +41,16 @@ #include "test_precomp.hpp" +#define TEST_LEGACY +#include + +//#define DEBUG_TEST +#ifdef DEBUG_TEST +#include +#endif + namespace opencv_test { namespace { +//using namespace cv::tracking; #define TESTSET_NAMES testing::Values("david","dudek","faceocc2") @@ -49,373 +58,10 @@ const string TRACKING_DIR = "tracking"; const string FOLDER_IMG = "data"; const string FOLDER_OMIT_INIT = "initOmit"; -/* - * The Evaluation Methodologies are partially based on: - * ==================================================================================================================== - * [OTB] Y. Wu, J. Lim, and M.-H. Yang, "Online object tracking: A benchmark," in Computer Vision and Pattern Recognition (CVPR), 2013 - * - */ - -enum BBTransformations -{ - NoTransform = 0, - CenterShiftLeft = 1, - CenterShiftRight = 2, - CenterShiftUp = 3, - CenterShiftDown = 4, - CornerShiftTopLeft = 5, - CornerShiftTopRight = 6, - CornerShiftBottomLeft = 7, - CornerShiftBottomRight = 8, - Scale_0_8 = 9, - Scale_0_9 = 10, - Scale_1_1 = 11, - Scale_1_2 = 12 -}; - -class TrackerTest -{ - public: - - TrackerTest(Ptr _tracker, string _video, float _distanceThreshold, - float _overlapThreshold, int _shift = NoTransform, int _segmentIdx = 1, int _numSegments = 10 ); - virtual ~TrackerTest(); - virtual void run(); - - protected: - void checkDataTest(); +// Check used "cmake" version in case of errors +// Check compiler command line options for /modules include +#include "video/test/test_trackers.impl.hpp" - void distanceAndOvrerlapTest(); - - Ptr tracker; - string video; - std::vector bbs; - int startFrame; - string suffix; - string prefix; - float overlapThreshold; - float distanceThreshold; - int segmentIdx; - int shift; - int numSegments; - - int gtStartFrame; - int endFrame; - vector validSequence; - - private: - float calcDistance( Rect a, Rect b ); - float calcOverlap( Rect a, Rect b ); - Rect applyShift(Rect bb); - std::vector splitString( std::string s, std::string delimiter ); - -}; - -TrackerTest::TrackerTest(Ptr _tracker, string _video, float _distanceThreshold, - float _overlapThreshold, int _shift, int _segmentIdx, int _numSegments ) : - tracker( _tracker ), - video( _video ), - overlapThreshold( _overlapThreshold ), - distanceThreshold( _distanceThreshold ), - segmentIdx(_segmentIdx), - shift(_shift), - numSegments(_numSegments) -{ -} - -TrackerTest::~TrackerTest() -{ - -} - -std::vector TrackerTest::splitString( std::string s, std::string delimiter ) -{ - std::vector token; - size_t pos = 0; - while ( ( pos = s.find( delimiter ) ) != std::string::npos ) - { - token.push_back( s.substr( 0, pos ) ); - s.erase( 0, pos + delimiter.length() ); - } - token.push_back( s ); - return token; -} - -float TrackerTest::calcDistance( Rect a, Rect b ) -{ - Point2f p_a( (float)(a.x + a.width / 2), (float)(a.y + a.height / 2) ); - Point2f p_b( (float)(b.x + b.width / 2), (float)(b.y + b.height / 2) ); - return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) ); -} - -float TrackerTest::calcOverlap( Rect a, Rect b ) -{ - float rectIntersectionArea = (float)(a & b).area(); - return rectIntersectionArea / (a.area() + b.area() - rectIntersectionArea); -} - -Rect TrackerTest::applyShift(Rect bb) -{ - Point center( bb.x + ( bb.width / 2 ), bb.y + ( bb.height / 2 ) ); - - int xLimit = bb.x + bb.width - 1; - int yLimit = bb.y + bb.height - 1; - - int h = 0; - int w = 0; - float ratio = 1.0; - - switch ( shift ) - { - case CenterShiftLeft: - bb.x = bb.x - (int)ceil( 0.1 * bb.width ); - break; - case CenterShiftRight: - bb.x = bb.x + (int)ceil( 0.1 * bb.width ); - break; - case CenterShiftUp: - bb.y = bb.y - (int)ceil( 0.1 * bb.height ); - break; - case CenterShiftDown: - bb.y = bb.y + (int)ceil( 0.1 * bb.height ); - break; - case CornerShiftTopLeft: - bb.x = (int)cvRound( bb.x - 0.1 * bb.width ); - bb.y = (int)cvRound( bb.y - 0.1 * bb.height ); - - bb.width = xLimit - bb.x + 1; - bb.height = yLimit - bb.y + 1; - break; - case CornerShiftTopRight: - xLimit = (int)cvRound( xLimit + 0.1 * bb.width ); - - bb.y = (int)cvRound( bb.y - 0.1 * bb.height ); - bb.width = xLimit - bb.x + 1; - bb.height = yLimit - bb.y + 1; - break; - case CornerShiftBottomLeft: - bb.x = (int)cvRound( bb.x - 0.1 * bb.width ); - yLimit = (int)cvRound( yLimit + 0.1 * bb.height ); - - bb.width = xLimit - bb.x + 1; - bb.height = yLimit - bb.y + 1; - break; - case CornerShiftBottomRight: - xLimit = (int)cvRound( xLimit + 0.1 * bb.width ); - yLimit = (int)cvRound( yLimit + 0.1 * bb.height ); - - bb.width = xLimit - bb.x + 1; - bb.height = yLimit - bb.y + 1; - break; - case Scale_0_8: - ratio = 0.8f; - w = (int)(ratio * bb.width); - h = (int)(ratio * bb.height); - - bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); - break; - case Scale_0_9: - ratio = 0.9f; - w = (int)(ratio * bb.width); - h = (int)(ratio * bb.height); - - bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); - break; - case 11: - //scale 1.1 - ratio = 1.1f; - w = (int)(ratio * bb.width); - h = (int)(ratio * bb.height); - - bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); - break; - case 12: - //scale 1.2 - ratio = 1.2f; - w = (int)(ratio * bb.width); - h = (int)(ratio * bb.height); - - bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); - break; - default: - break; - } - - return bb; -} - -void TrackerTest::distanceAndOvrerlapTest() -{ - Mat frame; - bool initialized = false; - - int fc = ( startFrame - gtStartFrame ); - - bbs.at( fc ) = applyShift(bbs.at( fc )); - Rect currentBBi = bbs.at( fc ); - Rect2d currentBB(currentBBi); - float sumDistance = 0; - float sumOverlap = 0; - - string folder = cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - - VideoCapture c; - c.open( folder + "/" + video + ".webm" ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); - - for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) - { - c >> frame; - - if( frame.empty() ) - { - break; - } - if( !initialized ) - { - if( !tracker->init( frame, currentBB ) ) - { - FAIL()<< "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if( initialized ) - { - if( frameCounter >= (int) bbs.size() ) - break; - tracker->update( frame, currentBB ); - } - float curDistance = calcDistance( currentBB, bbs.at( fc ) ); - float curOverlap = calcOverlap( currentBB, bbs.at( fc ) ); - - sumDistance += curDistance; - sumOverlap += curOverlap; - fc++; - } - - float meanDistance = sumDistance / (endFrame - startFrame); - float meanOverlap = sumOverlap / (endFrame - startFrame); - - if( meanDistance > distanceThreshold ) - { - FAIL()<< "Incorrect distance: curr = " << meanDistance << ", max = " << distanceThreshold << endl; - return; - } - - if( meanOverlap < overlapThreshold ) - { - FAIL()<< "Incorrect overlap: curr = " << meanOverlap << ", min = " << overlapThreshold << endl; - return; - } -} - -void TrackerTest::checkDataTest() -{ - - FileStorage fs; - fs.open( cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + video + ".yml", FileStorage::READ ); - fs["start"] >> startFrame; - fs["prefix"] >> prefix; - fs["suffix"] >> suffix; - fs.release(); - - string gtFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/gt.txt"; - std::ifstream gt; - //open the ground truth - gt.open( gtFile.c_str() ); - if( !gt.is_open() ) - { - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - } - string line; - int bbCounter = 0; - while ( getline( gt, line ) ) - { - bbCounter++; - } - gt.close(); - - int seqLength = bbCounter; - for ( int i = startFrame; i < seqLength; i++ ) - { - validSequence.push_back( i ); - } - - //exclude from the images sequence, the frames where the target is occluded or out of view - string omitFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_OMIT_INIT + "/" + video + ".txt"; - std::ifstream omit; - omit.open( omitFile.c_str() ); - if( omit.is_open() ) - { - string omitLine; - while ( getline( omit, omitLine ) ) - { - vector tokens = splitString( omitLine, " " ); - int s_start = atoi( tokens.at( 0 ).c_str() ); - int s_end = atoi( tokens.at( 1 ).c_str() ); - for ( int k = s_start; k <= s_end; k++ ) - { - std::vector::iterator position = std::find( validSequence.begin(), validSequence.end(), k ); - if( position != validSequence.end() ) - validSequence.erase( position ); - } - } - } - omit.close(); - gtStartFrame = startFrame; - //compute the start and the and for each segment - int numFrame = (int)(validSequence.size() / numSegments); - startFrame += ( segmentIdx - 1 ) * numFrame; - endFrame = startFrame + numFrame; - - std::ifstream gt2; - //open the ground truth - gt2.open( gtFile.c_str() ); - if( !gt2.is_open() ) - { - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - } - string line2; - int bbCounter2 = 0; - while ( getline( gt2, line2 ) ) - { - vector tokens = splitString( line2, "," ); - Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ); - if( tokens.size() != 4 ) - { - FAIL()<< "Incorrect ground truth file"; - } - - bbs.push_back( bb ); - bbCounter2++; - } - gt2.close(); - - if( segmentIdx == numSegments ) - endFrame = (int)bbs.size(); -} - -void TrackerTest::run() -{ - srand( 1 ); - - SCOPED_TRACE( "A" ); - - if( !tracker ) - { - FAIL()<< "Error in the instantiation of the tracker" << endl; - return; - } - - checkDataTest(); - - //check for failure - if( ::testing::Test::HasFatalFailure() ) - return; - - distanceAndOvrerlapTest(); -} /****************************************************************************************\ * Tests registrations * @@ -433,167 +79,158 @@ PARAM_TEST_CASE(DistanceAndOverlap, string) TEST_P(DistanceAndOverlap, MedianFlow) { - TrackerTest test( TrackerMedianFlow::create(), dataset, 35, .5f, NoTransform, 1, 1); + TrackerTest test(legacy::TrackerMedianFlow::create(), dataset, 35, .5f, NoTransform, 1, 1); test.run(); } -TEST_P(DistanceAndOverlap, MIL) +TEST_P(DistanceAndOverlap, Boosting) { - TrackerTest test( TrackerMIL::create(), dataset, 30, .65f, NoTransform); + TrackerTest test(legacy::TrackerBoosting::create(), dataset, 70, .7f, NoTransform); test.run(); } -TEST_P(DistanceAndOverlap, Boosting) +TEST_P(DistanceAndOverlap, KCF) { - TrackerTest test( TrackerBoosting::create(), dataset, 70, .7f, NoTransform); + TrackerTest test(TrackerKCF::create(), dataset, 20, .35f, NoTransform, 5); test.run(); } - -TEST_P(DistanceAndOverlap, KCF) +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, KCF_legacy) { - TrackerTest test( TrackerKCF::create(), dataset, 20, .35f, NoTransform, 5); + TrackerTest test(legacy::TrackerKCF::create(), dataset, 20, .35f, NoTransform, 5); test.run(); } +#endif TEST_P(DistanceAndOverlap, TLD) { - TrackerTest test( TrackerTLD::create(), dataset, 40, .45f, NoTransform); + TrackerTest test(legacy::TrackerTLD::create(), dataset, 40, .45f, NoTransform); test.run(); } TEST_P(DistanceAndOverlap, MOSSE) { - TrackerTest test( TrackerMOSSE::create(), dataset, 22, .7f, NoTransform); + TrackerTest test(legacy::TrackerMOSSE::create(), dataset, 22, .7f, NoTransform); test.run(); } TEST_P(DistanceAndOverlap, CSRT) { - TrackerTest test( TrackerCSRT::create(), dataset, 22, .7f, NoTransform); + TrackerTest test(TrackerCSRT::create(), dataset, 22, .7f, NoTransform); test.run(); } +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, CSRT_legacy) +{ + TrackerTest test(legacy::TrackerCSRT::create(), dataset, 22, .7f, NoTransform); + test.run(); +} +#endif /***************************************************************************************/ //Tests with shifted initial window TEST_P(DistanceAndOverlap, Shifted_Data_MedianFlow) { - TrackerTest test( TrackerMedianFlow::create(), dataset, 80, .2f, CenterShiftLeft, 1, 1); + TrackerTest test(legacy::TrackerMedianFlow::create(), dataset, 80, .2f, CenterShiftLeft, 1, 1); test.run(); } -TEST_P(DistanceAndOverlap, Shifted_Data_MIL) +TEST_P(DistanceAndOverlap, Shifted_Data_Boosting) { - TrackerTest test( TrackerMIL::create(), dataset, 30, .6f, CenterShiftLeft); + TrackerTest test(legacy::TrackerBoosting::create(), dataset, 80, .65f, CenterShiftLeft); test.run(); } -TEST_P(DistanceAndOverlap, Shifted_Data_Boosting) +TEST_P(DistanceAndOverlap, Shifted_Data_KCF) { - TrackerTest test( TrackerBoosting::create(), dataset, 80, .65f, CenterShiftLeft); + TrackerTest test(TrackerKCF::create(), dataset, 20, .4f, CenterShiftLeft, 5); test.run(); } - -TEST_P(DistanceAndOverlap, Shifted_Data_KCF) +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Shifted_Data_KCF_legacy) { - TrackerTest test( TrackerKCF::create(), dataset, 20, .4f, CenterShiftLeft, 5); + TrackerTest test(legacy::TrackerKCF::create(), dataset, 20, .4f, CenterShiftLeft, 5); test.run(); } +#endif TEST_P(DistanceAndOverlap, Shifted_Data_TLD) { - TrackerTest test( TrackerTLD::create(), dataset, 30, .35f, CenterShiftLeft); + TrackerTest test(legacy::TrackerTLD::create(), dataset, 30, .35f, CenterShiftLeft); test.run(); } TEST_P(DistanceAndOverlap, Shifted_Data_MOSSE) { - TrackerTest test( TrackerMOSSE::create(), dataset, 13, .69f, CenterShiftLeft); + TrackerTest test(legacy::TrackerMOSSE::create(), dataset, 13, .69f, CenterShiftLeft); test.run(); } TEST_P(DistanceAndOverlap, Shifted_Data_CSRT) { - TrackerTest test( TrackerCSRT::create(), dataset, 13, .69f, CenterShiftLeft); + TrackerTest test(TrackerCSRT::create(), dataset, 13, .69f, CenterShiftLeft); test.run(); } -/***************************************************************************************/ -//Tests with scaled initial window -TEST_P(DistanceAndOverlap, Scaled_Data_MedianFlow) +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Shifted_Data_CSRT_legacy) { - TrackerTest test( TrackerMedianFlow::create(), dataset, 25, .5f, Scale_1_1, 1, 1); + TrackerTest test(legacy::TrackerCSRT::create(), dataset, 13, .69f, CenterShiftLeft); test.run(); } +#endif -TEST_P(DistanceAndOverlap, Scaled_Data_MIL) +/***************************************************************************************/ +//Tests with scaled initial window +TEST_P(DistanceAndOverlap, Scaled_Data_MedianFlow) { - TrackerTest test( TrackerMIL::create(), dataset, 30, .7f, Scale_1_1); + TrackerTest test(legacy::TrackerMedianFlow::create(), dataset, 25, .5f, Scale_1_1, 1, 1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_Boosting) { - TrackerTest test( TrackerBoosting::create(), dataset, 80, .7f, Scale_1_1); + TrackerTest test(legacy::TrackerBoosting::create(), dataset, 80, .7f, Scale_1_1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_KCF) { - TrackerTest test( TrackerKCF::create(), dataset, 20, .4f, Scale_1_1, 5); + TrackerTest test(TrackerKCF::create(), dataset, 20, .4f, Scale_1_1, 5); test.run(); } - -TEST_P(DistanceAndOverlap, Scaled_Data_TLD) +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Scaled_Data_KCF_legacy) { - TrackerTest test( TrackerTLD::create(), dataset, 30, .45f, Scale_1_1); + TrackerTest test(legacy::TrackerKCF::create(), dataset, 20, .4f, Scale_1_1, 5); test.run(); } +#endif - -TEST_P(DistanceAndOverlap, DISABLED_GOTURN) +TEST_P(DistanceAndOverlap, Scaled_Data_TLD) { - TrackerTest test(TrackerGOTURN::create(), dataset, 18, .5f, NoTransform); + TrackerTest test(legacy::TrackerTLD::create(), dataset, 30, .45f, Scale_1_1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_MOSSE) { - TrackerTest test( TrackerMOSSE::create(), dataset, 22, 0.69f, Scale_1_1, 1); + TrackerTest test(legacy::TrackerMOSSE::create(), dataset, 22, 0.69f, Scale_1_1, 1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_CSRT) { - TrackerTest test( TrackerCSRT::create(), dataset, 22, 0.69f, Scale_1_1, 1); + TrackerTest test(TrackerCSRT::create(), dataset, 22, 0.69f, Scale_1_1, 1); test.run(); } - -TEST(GOTURN, memory_usage) +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Scaled_Data_CSRT_legacy) { - cv::Rect2d roi(145, 70, 85, 85); - cv::Mat frame; - - std::string model = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.prototxt"); - std::string weights = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.caffemodel", false); - cv::TrackerGOTURN::Params params; - params.modelTxt = model; - params.modelBin = weights; - cv::Ptr tracker = cv::TrackerGOTURN::create(params); - string inputVideo = cvtest::findDataFile("tracking/david/data/david.webm"); - cv::VideoCapture video(inputVideo); - - video >> frame; - tracker->init(frame, roi); - string ground_truth_bb; - for (int nframes = 0; nframes < 15; ++nframes) - { - std::cout << "Frame: " << nframes << std::endl; - video >> frame; - tracker->update(frame, roi); - std::cout << "Predicted ROI: " << roi << std::endl; - } + TrackerTest test(TrackerCSRT::create(), dataset, 22, 0.69f, Scale_1_1, 1); + test.run(); } +#endif -INSTANTIATE_TEST_CASE_P( Tracking, DistanceAndOverlap, TESTSET_NAMES); +INSTANTIATE_TEST_CASE_P(Tracking, DistanceAndOverlap, TESTSET_NAMES); }} // namespace -/* End of file. */ diff --git a/modules/tracking/test/test_ukf.cpp b/modules/tracking/test/test_ukf.cpp index 60196ee1d36..c8ebe3edc1d 100644 --- a/modules/tracking/test/test_ukf.cpp +++ b/modules/tracking/test/test_ukf.cpp @@ -43,7 +43,7 @@ #include "opencv2/tracking/kalman_filters.hpp" namespace opencv_test { namespace { -using namespace cv::tracking; +using namespace cv::detail; // In this two tests Unscented Kalman Filter are applied to the dynamic system from example "The reentry problem" from // "A New Extension of the Kalman Filter to Nonlinear Systems" by Simon J. Julier and Jeffrey K. Uhlmann. diff --git a/modules/tracking/tutorials/tutorial_multitracker.markdown b/modules/tracking/tutorials/tutorial_multitracker.markdown index f9c7605a53d..0dce97fc4b7 100644 --- a/modules/tracking/tutorials/tutorial_multitracker.markdown +++ b/modules/tracking/tutorials/tutorial_multitracker.markdown @@ -38,11 +38,11 @@ Explanation You can add all tracked objects at once to the MultiTracker as shown in the code. In this case, all objects will be tracked using same tracking algorithm as specified in decaration of MultiTracker object. If you want to use different tracker algorithms for each tracked object, - You should add the tracked objects one by one and specify their tracking algorithm using the variant of @ref cv::MultiTracker::add. - @sa cv::MultiTracker::add( const String& trackerType, const Mat& image, const Rect2d& boundingBox ) + You should add the tracked objects one by one and specify their tracking algorithm using the variant of @ref cv::legacy::MultiTracker::add. + @sa cv::legacy::MultiTracker::add( const String& trackerType, const Mat& image, const Rect2d& boundingBox ) -# **Obtaining the result** @snippet tracking/samples/tutorial_multitracker.cpp result - You can access the result from the public variable @ref cv::MultiTracker::objects provided by the MultiTracker class as shown in the code. + You can access the result from the public variable @ref cv::legacy::MultiTracker::objects provided by the MultiTracker class as shown in the code. diff --git a/modules/viz/include/opencv2/viz/vizcore.hpp b/modules/viz/include/opencv2/viz/vizcore.hpp index 85b75131d68..7579ddc3ee7 100644 --- a/modules/viz/include/opencv2/viz/vizcore.hpp +++ b/modules/viz/include/opencv2/viz/vizcore.hpp @@ -92,12 +92,7 @@ namespace cv @note If the window with that name already exists, that window is returned. Otherwise, new window is created with the given name, and it is returned. - @note Window names are automatically prefixed by "Viz - " if it is not done by the user. - @code - /// window and window_2 are the same windows. - viz::Viz3d window = viz::getWindowByName("myWindow"); - viz::Viz3d window_2 = viz::getWindowByName("Viz - myWindow"); - @endcode + */ CV_EXPORTS Viz3d getWindowByName(const String &window_name); diff --git a/modules/viz/tutorials/transformations/transformations.markdown b/modules/viz/tutorials/transformations/transformations.markdown index 44d2cd47d91..b5c73db8968 100644 --- a/modules/viz/tutorials/transformations/transformations.markdown +++ b/modules/viz/tutorials/transformations/transformations.markdown @@ -84,8 +84,8 @@ Results -# Here is the result from the camera point of view. - ![](images/camera_view_point.png) +![](images/camera_view_point.png) -# Here is the result from global point of view. - ![](images/global_view_point.png) +![](images/global_view_point.png) \ No newline at end of file diff --git a/modules/xfeatures2d/doc/xfeatures2d.bib b/modules/xfeatures2d/doc/xfeatures2d.bib index b88be8bfe1e..6337a606d17 100644 --- a/modules/xfeatures2d/doc/xfeatures2d.bib +++ b/modules/xfeatures2d/doc/xfeatures2d.bib @@ -80,6 +80,20 @@ @article{Mikolajczyk2004 publisher = {Springer} } +@ARTICLE{Najman2014, + author={Y. {Xu} and P. {Monasse} and T. {Géraud} and L. {Najman}}, + journal={IEEE Transactions on Image Processing}, + title={Tree-Based Morse Regions: A Topological Approach to Local Feature Detection}, + year={2014}, + volume={23}, + number={12}, + pages={5612-5625}, + abstract={This paper introduces a topological approach to local invariant feature detection motivated by Morse theory. We use the critical points of the graph of the intensity image, revealing directly the topology information as initial interest points. Critical points are selected from what we call a tree-based shape-space. In particular, they are selected from both the connected components of the upper level sets of the image (the Max-tree) and those of the lower level sets (the Min-tree). They correspond to specific nodes on those two trees: 1) to the leaves (extrema) and 2) to the nodes having bifurcation (saddle points). We then associate to each critical point the largest region that contains it and is topologically equivalent in its tree. We call such largest regions the tree-based Morse regions (TBMRs). The TBMR can be seen as a variant of maximally stable extremal region (MSER), which are contrasted regions. Contrarily to MSER, TBMR relies only on topological information and thus fully inherit the invariance properties of the space of shapes (e.g., invariance to affine contrast changes and covariance to continuous transformations). In particular, TBMR extracts the regions independently of the contrast, which makes it truly contrast invariant. Furthermore, it is quasi-parameter free. TBMR extraction is fast, having the same complexity as MSER. Experimentally, TBMR achieves a repeatability on par with state-of-the-art methods, but obtains a significantly higher number of features. Both the accuracy and robustness of TBMR are demonstrated by applications to image registration and 3D reconstruction.}, + keywords={feature extraction;image reconstruction;image registration;trees (mathematics);tree-based Morse regions;topological approach;local invariant feature detection;Morse theory;intensity image;initial interest points;critical points;tree-based shape-space;upper level image sets;Max-tree;lower level sets;Min-tree;saddle points;bifurcation;maximally stable extremal region variant;MSER;topological information;TBMR extraction;3D reconstruction;image registration;Feature extraction;Detectors;Shape;Time complexity;Level set;Three-dimensional displays;Image registration;Min/Max tree;local features;affine region detectors;image registration;3D reconstruction;Min/Max tree;local features;affine region detectors;image registration;3D reconstruction}, + doi={10.1109/TIP.2014.2364127}, + ISSN={1941-0042}, + month={Dec},} + @article{Simonyan14, author = {Simonyan, K. and Vedaldi, A. and Zisserman, A.}, title = {Learning Local Feature Descriptors Using Convex Optimisation}, @@ -126,4 +140,4 @@ @incollection{LUCID pages = {1--9} year = {2012} publisher = {NIPS} -} +} \ No newline at end of file diff --git a/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp b/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp index b2eda6f643e..103dbe8e78c 100644 --- a/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp +++ b/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp @@ -901,7 +901,7 @@ class CV_EXPORTS_W HarrisLaplaceFeatureDetector : public Feature2D * The interface is equivalent to @ref Feature2D, adding operations for * @ref Elliptic_KeyPoint "Elliptic_KeyPoints" instead of @ref KeyPoint "KeyPoints". */ -class CV_EXPORTS AffineFeature2D : public Feature2D +class CV_EXPORTS_W AffineFeature2D : public Feature2D { public: /** @@ -945,6 +945,40 @@ class CV_EXPORTS AffineFeature2D : public Feature2D bool useProvidedKeypoints=false ) = 0; }; +/** +@brief Class implementing the Tree Based Morse Regions (TBMR) as described in +@cite Najman2014 extended with scaled extraction ability. + +@param min_area prune areas smaller than minArea +@param max_area_relative prune areas bigger than maxArea = max_area_relative * +input_image_size +@param scale_factor scale factor for scaled extraction. +@param n_scales number of applications of the scale factor (octaves). + +@note This algorithm is based on Component Tree (Min/Max) as well as MSER but +uses a Morse-theory approach to extract features. + +Features are ellipses (similar to MSER, however a MSER feature can never be a +TBMR feature and vice versa). + +*/ +class CV_EXPORTS_W TBMR : public AffineFeature2D +{ +public: + CV_WRAP static Ptr create(int min_area = 60, + float max_area_relative = 0.01f, + float scale_factor = 1.25f, + int n_scales = -1); + + CV_WRAP virtual void setMinArea(int minArea) = 0; + CV_WRAP virtual int getMinArea() const = 0; + CV_WRAP virtual void setMaxAreaRelative(float maxArea) = 0; + CV_WRAP virtual float getMaxAreaRelative() const = 0; + CV_WRAP virtual void setScaleFactor(float scale_factor) = 0; + CV_WRAP virtual float getScaleFactor() const = 0; + CV_WRAP virtual void setNScales(int n_scales) = 0; + CV_WRAP virtual int getNScales() const = 0; +}; /** @brief Estimates cornerness for prespecified KeyPoints using the FAST algorithm diff --git a/modules/xfeatures2d/src/msd.cpp b/modules/xfeatures2d/src/msd.cpp index 637d992cbc9..9fe5ae8cf0a 100644 --- a/modules/xfeatures2d/src/msd.cpp +++ b/modules/xfeatures2d/src/msd.cpp @@ -55,78 +55,14 @@ University of Bologna, Open Perception */ #include "precomp.hpp" +#include "msd_pyramid.hpp" #include namespace cv { namespace xfeatures2d { - /*! - MSD Image Pyramid. - */ - class MSDImagePyramid - { - // Multi-threaded construction of the scale-space pyramid - struct MSDImagePyramidBuilder : ParallelLoopBody - { - - MSDImagePyramidBuilder(const cv::Mat& _im, std::vector* _m_imPyr, float _scaleFactor) - { - im = &_im; - m_imPyr = _m_imPyr; - scaleFactor = _scaleFactor; - - } - - void operator()(const Range& range) const CV_OVERRIDE - { - for (int lvl = range.start; lvl < range.end; lvl++) - { - float scale = 1 / std::pow(scaleFactor, (float) lvl); - (*m_imPyr)[lvl] = cv::Mat(cv::Size(cvRound(im->cols * scale), cvRound(im->rows * scale)), im->type()); - cv::resize(*im, (*m_imPyr)[lvl], cv::Size((*m_imPyr)[lvl].cols, (*m_imPyr)[lvl].rows), 0.0, 0.0, cv::INTER_AREA); - } - } - const cv::Mat* im; - std::vector* m_imPyr; - float scaleFactor; - }; - - public: - MSDImagePyramid(const cv::Mat &im, const int nLevels, const float scaleFactor = 1.6f); - ~MSDImagePyramid(); - - const std::vector getImPyr() const - { - return m_imPyr; - }; - - private: - - std::vector m_imPyr; - int m_nLevels; - float m_scaleFactor; - }; - - MSDImagePyramid::MSDImagePyramid(const cv::Mat & im, const int nLevels, const float scaleFactor) - { - m_nLevels = nLevels; - m_scaleFactor = scaleFactor; - m_imPyr.clear(); - m_imPyr.resize(nLevels); - - m_imPyr[0] = im.clone(); - - if (m_nLevels > 1) - { - parallel_for_(Range(1, nLevels), MSDImagePyramidBuilder(im, &m_imPyr, scaleFactor)); - } - } - - MSDImagePyramid::~MSDImagePyramid() - { - } /*! MSD Implementation. diff --git a/modules/xfeatures2d/src/msd_pyramid.hpp b/modules/xfeatures2d/src/msd_pyramid.hpp new file mode 100644 index 00000000000..9fc3243a320 --- /dev/null +++ b/modules/xfeatures2d/src/msd_pyramid.hpp @@ -0,0 +1,77 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef __OPENCV_XFEATURES2D_MSD_PYRAMID_HPP__ +#define __OPENCV_XFEATURES2D_MSD_PYRAMID_HPP__ + +#include "precomp.hpp" + +namespace cv +{ +namespace xfeatures2d +{ +/*! + MSD Image Pyramid. + */ +class MSDImagePyramid +{ + // Multi-threaded construction of the scale-space pyramid + struct MSDImagePyramidBuilder : ParallelLoopBody + { + + MSDImagePyramidBuilder(const cv::Mat& _im, std::vector* _m_imPyr, float _scaleFactor) + { + im = &_im; + m_imPyr = _m_imPyr; + scaleFactor = _scaleFactor; + + } + + void operator()(const Range& range) const CV_OVERRIDE + { + for (int lvl = range.start; lvl < range.end; lvl++) + { + float scale = 1 / std::pow(scaleFactor, (float) lvl); + (*m_imPyr)[lvl] = cv::Mat(cv::Size(cvRound(im->cols * scale), cvRound(im->rows * scale)), im->type()); + cv::resize(*im, (*m_imPyr)[lvl], cv::Size((*m_imPyr)[lvl].cols, (*m_imPyr)[lvl].rows), 0.0, 0.0, cv::INTER_AREA); + } + } + const cv::Mat* im; + std::vector* m_imPyr; + float scaleFactor; + }; + +public: + + MSDImagePyramid(const cv::Mat &im, const int nLevels, const float scaleFactor = 1.6f) + { + m_nLevels = nLevels; + m_scaleFactor = scaleFactor; + m_imPyr.clear(); + m_imPyr.resize(nLevels); + + m_imPyr[0] = im.clone(); + + if (m_nLevels > 1) + { + parallel_for_(Range(1, nLevels), MSDImagePyramidBuilder(im, &m_imPyr, scaleFactor)); + } + } + ~MSDImagePyramid() {}; + + const std::vector getImPyr() const + { + return m_imPyr; + }; + +private: + + std::vector m_imPyr; + int m_nLevels; + float m_scaleFactor; +}; +} +} + +#endif \ No newline at end of file diff --git a/modules/xfeatures2d/src/tbmr.cpp b/modules/xfeatures2d/src/tbmr.cpp new file mode 100644 index 00000000000..36a0a538b7e --- /dev/null +++ b/modules/xfeatures2d/src/tbmr.cpp @@ -0,0 +1,584 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level +// directory of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" +#include "msd_pyramid.hpp" + +namespace cv +{ +namespace xfeatures2d +{ + +class TBMR_Impl CV_FINAL : public TBMR +{ + public: + struct Params + { + Params(int _min_area = 60, float _max_area_relative = 0.01, + float _scale = 1.5, int _n_scale = -1) + { + CV_Assert(_min_area >= 0); + CV_Assert(_max_area_relative >= + std::numeric_limits::epsilon()); + + minArea = _min_area; + maxAreaRelative = _max_area_relative; + scale = _scale; + n_scale = _n_scale; + } + + uint minArea; + float maxAreaRelative; + int n_scale; + float scale; + }; + + explicit TBMR_Impl(const Params &_params) : params(_params) {} + + virtual ~TBMR_Impl() CV_OVERRIDE {} + + virtual void setMinArea(int minArea) CV_OVERRIDE + { + params.minArea = std::max(minArea, 0); + } + int getMinArea() const CV_OVERRIDE { return params.minArea; } + + virtual void setMaxAreaRelative(float maxAreaRelative) CV_OVERRIDE + { + params.maxAreaRelative = + std::max(maxAreaRelative, std::numeric_limits::epsilon()); + } + virtual float getMaxAreaRelative() const CV_OVERRIDE + { + return params.maxAreaRelative; + } + virtual void setScaleFactor(float scale_factor) CV_OVERRIDE + { + params.scale = std::max(scale_factor, 1.f); + } + virtual float getScaleFactor() const CV_OVERRIDE { return params.scale; } + virtual void setNScales(int n_scales) CV_OVERRIDE + { + params.n_scale = n_scales; + } + virtual int getNScales() const CV_OVERRIDE { return params.n_scale; } + + virtual void detect(InputArray image, + CV_OUT std::vector &keypoints, + InputArray mask = noArray()) CV_OVERRIDE; + + virtual void detect(InputArray image, + CV_OUT std::vector &keypoints, + InputArray mask = noArray()) CV_OVERRIDE; + + virtual void + detectAndCompute(InputArray image, InputArray mask, + CV_OUT std::vector &keypoints, + OutputArray descriptors, + bool useProvidedKeypoints = false) CV_OVERRIDE; + + CV_INLINE uint zfindroot(uint *parent, uint p) + { + if (parent[p] == p) + return p; + else + return parent[p] = zfindroot(parent, parent[p]); + } + + // Calculate the Component tree. Based on the order of S, it will be a + // min or max tree. + void calcMinMaxTree(Mat ima) + { + int rs = ima.rows; + int cs = ima.cols; + uint imSize = (uint)rs * cs; + + std::array offsets = { + -ima.cols, -1, 1, ima.cols + }; // {-1,0}, {0,-1}, {0,1}, {1,0} yx + std::array offsetsv = { Vec2i(0, -1), Vec2i(-1, 0), + Vec2i(1, 0), Vec2i(0, 1) }; // xy + AutoBuffer zparb(imSize); + AutoBuffer rootb(imSize); + AutoBuffer rankb(imSize); + memset(rankb.data(), 0, imSize * sizeof(uint)); + uint* zpar = zparb.data(); + uint *root = rootb.data(); + uint *rank = rankb.data(); + parent = Mat(rs, cs, CV_32S); // unsigned + AutoBuffer dejaVub(imSize); + memset(dejaVub.data(), 0, imSize * sizeof(bool)); + bool* dejaVu = dejaVub.data(); + + const uint *S_ptr = S.ptr(); + uint *parent_ptr = parent.ptr(); + Vec *imaAttribute = imaAttributes.ptr>(); + + for (int i = imSize - 1; i >= 0; --i) + { + uint p = S_ptr[i]; + + Vec2i idx_p(p % cs, p / cs); + // make set + { + parent_ptr[p] = p; + zpar[p] = p; + root[p] = p; + dejaVu[p] = true; + imaAttribute[p][0] = 1; // area + imaAttribute[p][1] = idx_p[0]; // sum_x + imaAttribute[p][2] = idx_p[1]; // sum_y + imaAttribute[p][3] = idx_p[0] * idx_p[1]; // sum_xy + imaAttribute[p][4] = idx_p[0] * idx_p[0]; // sum_xx + imaAttribute[p][5] = idx_p[1] * idx_p[1]; // sum_yy + } + + uint x = p; // zpar of p + for (unsigned k = 0; k < offsets.size(); ++k) + { + uint q = p + offsets[k]; + + Vec2i q_idx = idx_p + offsetsv[k]; + bool inBorder = q_idx[0] >= 0 && q_idx[0] < ima.cols && + q_idx[1] >= 0 && + q_idx[1] < ima.rows; // filter out border cases + + if (inBorder && dejaVu[q]) // remove first check + // obsolete + { + uint r = zfindroot(zpar, q); + if (r != x) // make union + { + parent_ptr[root[r]] = p; + // accumulate information + imaAttribute[p][0] += imaAttribute[root[r]][0]; // area + imaAttribute[p][1] += imaAttribute[root[r]][1]; // sum_x + imaAttribute[p][2] += imaAttribute[root[r]][2]; // sum_y + imaAttribute[p][3] += + imaAttribute[root[r]][3]; // sum_xy + imaAttribute[p][4] += + imaAttribute[root[r]][4]; // sum_xx + imaAttribute[p][5] += + imaAttribute[root[r]][5]; // sum_yy + + if (rank[x] < rank[r]) + { + // we merge p to r + zpar[x] = r; + root[r] = p; + x = r; + } + else if (rank[r] < rank[p]) + { + // merge r to p + zpar[r] = p; + } + else + { + // same height + zpar[r] = p; + rank[p] += 1; + } + } + } + } + } + } + + void calculateTBMRs(const Mat &image, std::vector &tbmrs, + const Mat &mask, float scale, int octave) + { + uint imSize = image.cols * image.rows; + uint maxArea = + static_cast(params.maxAreaRelative * imSize * scale); + uint minArea = static_cast(params.minArea * scale); + + if (parent.empty() || parent.size != image.size) + parent = Mat(image.rows, image.cols, CV_32S); + + if (imaAttributes.empty() || imaAttributes.size != image.size) + imaAttributes = Mat(image.rows, image.cols, CV_32SC(6)); + + calcMinMaxTree(image); + + const Vec *imaAttribute = + imaAttributes.ptr>(); + const uint8_t *ima_ptr = image.ptr(); + const uint *S_ptr = S.ptr(); + uint *parent_ptr = parent.ptr(); + + // canonization + for (uint i = 0; i < imSize; ++i) + { + uint p = S_ptr[i]; + uint q = parent_ptr[p]; + if (ima_ptr[parent_ptr[q]] == ima_ptr[q]) + parent_ptr[p] = parent_ptr[q]; + } + + // TBMRs extraction + //------------------------------------------------------------------------ + // small variant of the given algorithm in the paper. For each + // critical node having more than one child, we check if the + // largest region containing this node without any change of + // topology is above its parent, if not, discard this critical + // node. + // + // note also that we do not select the critical nodes themselves + // as final TBMRs + //-------------------------------------------------------------------------- + + AutoBuffer numSonsb(imSize); + memset(numSonsb.data(), 0, imSize * sizeof(uint)); + uint* numSons = numSonsb.data(); + uint vecNodesSize = imaAttribute[S_ptr[0]][0]; // area + AutoBuffer vecNodesb(vecNodesSize); + memset(vecNodesb.data(), 0, vecNodesSize * sizeof(uint)); + uint *vecNodes = vecNodesb.data(); // area + uint numNodes = 0; + + // leaf to root propagation to select the canonized nodes + for (int i = imSize - 1; i >= 0; --i) + { + uint p = S_ptr[i]; + if (parent_ptr[p] == p || ima_ptr[p] != ima_ptr[parent_ptr[p]]) + { + vecNodes[numNodes++] = p; + if (imaAttribute[p][0] >= minArea) // area + numSons[parent_ptr[p]]++; + } + } + + AutoBuffer isSeenb(imSize); + memset(isSeenb.data(), 0, imSize * sizeof(bool)); + bool *isSeen = isSeenb.data(); + + // parent of critical leaf node + AutoBuffer isParentofLeafb(imSize); + memset(isParentofLeafb.data(), 0, imSize * sizeof(bool)); + bool* isParentofLeaf = isParentofLeafb.data(); + + for (uint i = 0; i < vecNodesSize; i++) + { + uint p = vecNodes[i]; + if (numSons[p] == 0 && numSons[parent_ptr[p]] == 1) + isParentofLeaf[parent_ptr[p]] = true; + } + + uint numTbmrs = 0; + AutoBuffer vecTbmrsb(numNodes); + uint* vecTbmrs = vecTbmrsb.data(); + for (uint i = 0; i < vecNodesSize; i++) + { + uint p = vecNodes[i]; + if (numSons[p] == 1 && !isSeen[p] && imaAttribute[p][0] <= maxArea) + { + uint num_ancestors = 0; + uint pt = p; + uint po = pt; + while (numSons[pt] == 1 && imaAttribute[pt][0] <= maxArea) + { + isSeen[pt] = true; + num_ancestors++; + po = pt; + pt = parent_ptr[pt]; + } + if (!isParentofLeaf[p] || num_ancestors > 1) + { + vecTbmrs[numTbmrs++] = po; + } + } + } + // end of TBMRs extraction + //------------------------------------------------------------------------ + + // compute best fitting ellipses + //------------------------------------------------------------------------ + for (uint i = 0; i < numTbmrs; i++) + { + uint p = vecTbmrs[i]; + double area = static_cast(imaAttribute[p][0]); + double sum_x = static_cast(imaAttribute[p][1]); + double sum_y = static_cast(imaAttribute[p][2]); + double sum_xy = static_cast(imaAttribute[p][3]); + double sum_xx = static_cast(imaAttribute[p][4]); + double sum_yy = static_cast(imaAttribute[p][5]); + + // Barycenter: + double x = sum_x / area; + double y = sum_y / area; + + double i20 = sum_xx - area * x * x; + double i02 = sum_yy - area * y * y; + double i11 = sum_xy - area * x * y; + double n = i20 * i02 - i11 * i11; + if (n != 0) + { + double a = (i02 / n) * (area - 1) / 4; + double b = (-i11 / n) * (area - 1) / 4; + double c = (i20 / n) * (area - 1) / 4; + + // filter out some non meaningful ellipses + double a1 = a; + double b1 = b; + double c1 = c; + uint ai = 0; + uint bi = 0; + uint ci = 0; + if (a > 0) + { + if (a < 0.00005) + a1 = 0; + else if (a < 0.0001) + { + a1 = 0.0001; + } + else + { + ai = (uint)(10000 * a); + a1 = (double)ai / 10000; + } + } + else + { + if (a > -0.00005) + a1 = 0; + else if (a > -0.0001) + a1 = -0.0001; + else + { + ai = (uint)(10000 * (-a)); + a1 = -(double)ai / 10000; + } + } + + if (b > 0) + { + if (b < 0.00005) + b1 = 0; + else if (b < 0.0001) + { + b1 = 0.0001; + } + else + { + bi = (uint)(10000 * b); + b1 = (double)bi / 10000; + } + } + else + { + if (b > -0.00005) + b1 = 0; + else if (b > -0.0001) + b1 = -0.0001; + else + { + bi = (uint)(10000 * (-b)); + b1 = -(double)bi / 10000; + } + } + + if (c > 0) + { + if (c < 0.00005) + c1 = 0; + else if (c < 0.0001) + { + c1 = 0.0001; + } + else + { + ci = (uint)(10000 * c); + c1 = (double)ci / 10000; + } + } + else + { + if (c > -0.00005) + c1 = 0; + else if (c > -0.0001) + c1 = -0.0001; + else + { + ci = (uint)(10000 * (-c)); + c1 = -(double)ci / 10000; + } + } + double v = + (a1 + c1 - + std::sqrt(a1 * a1 + c1 * c1 + 4 * b1 * b1 - 2 * a1 * c1)) / + 2; + + double l1 = 1. / std::sqrt((a + c + + std::sqrt(a * a + c * c + + 4 * b * b - 2 * a * c)) / + 2); + double l2 = 1. / std::sqrt((a + c - + std::sqrt(a * a + c * c + + 4 * b * b - 2 * a * c)) / + 2); + double minAxL = std::min(l1, l2); + double majAxL = std::max(l1, l2); + + if (minAxL >= 1.5 && v != 0 && + (mask.empty() || + mask.at(cvRound(y), cvRound(x)) != 0)) + { + double theta = 0; + if (b == 0) + if (a < c) + theta = 0; + else + theta = CV_PI / 2.; + else + theta = CV_PI / 2. + 0.5 * std::atan2(2 * b, (a - c)); + + float size = (float)majAxL; + + // not sure if we should scale or not scale x,y,axes,size + // (as scale is stored in si) + Elliptic_KeyPoint ekp( + Point2f((float)x, (float)y) * scale, (float)theta, + cv::Size2f((float)majAxL, (float)minAxL) * scale, + size * scale, scale); + ekp.octave = octave; + tbmrs.push_back(ekp); + } + } + } + //--------------------------------------------- + } + + Mat tempsrc; + + // component tree representation (parent,S): see + // https://ieeexplore.ieee.org/document/6850018 + Mat parent; + Mat S; + // moments: compound type of: (area, x, y, xy, xx, yy) + Mat imaAttributes; + + Params params; +}; + +void TBMR_Impl::detect(InputArray _image, std::vector &keypoints, + InputArray _mask) +{ + std::vector kp; + detect(_image, kp, _mask); + keypoints.resize(kp.size()); + for (size_t i = 0; i < kp.size(); ++i) + keypoints[i] = kp[i]; +} + +void TBMR_Impl::detect(InputArray _image, + std::vector &keypoints, + InputArray _mask) +{ + Mat mask = _mask.getMat(); + Mat src = _image.getMat(); + + keypoints.clear(); + + if (src.empty()) + return; + + if (!mask.empty()) + { + CV_Assert(mask.type() == CV_8UC1); + CV_Assert(mask.size == src.size); + } + + if (!src.isContinuous()) + { + src.copyTo(tempsrc); + src = tempsrc; + } + + CV_Assert(src.depth() == CV_8U); + + if (src.channels() != 1) + cv::cvtColor(src, src, cv::COLOR_BGR2GRAY); + + int m_cur_n_scales = + params.n_scale > 0 + ? params.n_scale + : 1 /*todo calculate optimal scale factor from image size*/; + float m_scale_factor = params.scale; + + // track and eliminate duplicates introduced with multi scale position -> + // (size) + Mat dupl(src.rows / 4, src.cols / 4, CV_32F, cv::Scalar::all(0)); + float *dupl_ptr = dupl.ptr(); + + std::vector pyr; + MSDImagePyramid scaleSpacer(src, m_cur_n_scales, m_scale_factor); + pyr = scaleSpacer.getImPyr(); + + int oct = 0; + for (auto &s : pyr) + { + float scale = ((float)s.cols) / pyr.begin()->cols; + std::vector kpts; + + // append max tree tbmrs + sortIdx(s.reshape(1, 1), S, + SortFlags::SORT_ASCENDING | SortFlags::SORT_EVERY_ROW); + calculateTBMRs(s, kpts, mask, scale, oct); + + // reverse instead of sort + flip(S, S, -1); + calculateTBMRs(s, kpts, mask, scale, oct); + + if (oct == 0) + { + for (const auto &k : kpts) + { + dupl_ptr[(int)(k.pt.x / 4) + + (int)(k.pt.y / 4) * (src.cols / 4)] = k.size; + } + keypoints.insert(keypoints.end(), kpts.begin(), kpts.end()); + } + else + { + for (const auto &k : kpts) + { + float &sz = dupl_ptr[(int)(k.pt.x / 4) + + (int)(k.pt.y / 4) * (src.cols / 4)]; + // we hereby add only features that are at least 4 pixels away + // or have a significantly different size + if (std::abs(k.size - sz) / std::max(k.size, sz) >= 0.2f) + { + sz = k.size; + keypoints.push_back(k); + } + } + } + + oct++; + } +} + +void TBMR_Impl::detectAndCompute( + InputArray image, InputArray mask, + CV_OUT std::vector &keypoints, OutputArray descriptors, + bool useProvidedKeypoints) +{ + // We can use SIFT to compute descriptors for the extracted keypoints... + auto sift = SIFT::create(); + auto dac = AffineFeature2D::create(this, sift); + dac->detectAndCompute(image, mask, keypoints, descriptors, + useProvidedKeypoints); +} + +Ptr TBMR::create(int _min_area, float _max_area_relative, float _scale, + int _n_scale) +{ + return cv::makePtr( + TBMR_Impl::Params(_min_area, _max_area_relative, _scale, _n_scale)); +} + +} // namespace xfeatures2d +} // namespace cv \ No newline at end of file diff --git a/modules/xfeatures2d/test/test_features2d.cpp b/modules/xfeatures2d/test/test_features2d.cpp index 6a8d4b5db89..e79e50370bb 100644 --- a/modules/xfeatures2d/test/test_features2d.cpp +++ b/modules/xfeatures2d/test/test_features2d.cpp @@ -85,6 +85,12 @@ TEST( Features2d_Detector_Harris_Laplace_Affine, regression ) test.safe_run(); } +TEST(Features2d_Detector_TBMR_Affine, regression) +{ + CV_FeatureDetectorTest test("detector-tbmr-affine", TBMR::create()); + test.safe_run(); +} + /* * Descriptors */ diff --git a/modules/xfeatures2d/test/test_keypoints.cpp b/modules/xfeatures2d/test/test_keypoints.cpp index 45d50dfb4b6..28d125d0836 100644 --- a/modules/xfeatures2d/test/test_keypoints.cpp +++ b/modules/xfeatures2d/test/test_keypoints.cpp @@ -137,4 +137,10 @@ TEST(Features2d_Detector_Keypoints_MSDDetector, validation) test.safe_run(); } +TEST(Features2d_Detector_Keypoints_TBMRDetector, validation) +{ + CV_FeatureDetectorKeypointsTest test(xfeatures2d::TBMR::create()); + test.safe_run(); +} + }} // namespace diff --git a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp index 1df555865eb..4a33148be28 100644 --- a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp @@ -65,8 +65,9 @@ class CV_EXPORTS_W FastLineDetector : public Algorithm hysteresis procedure in Canny() @param _canny_th2 50 - Second threshold for hysteresis procedure in Canny() -@param _canny_aperture_size 3 - Aperturesize for the sobel - operator in Canny() +@param _canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). + If zero, Canny() is not applied and the input + image is taken as an edge image. @param _do_merge false - If true, incremental merging of segments will be perfomred */ diff --git a/modules/ximgproc/samples/fld_lines.cpp b/modules/ximgproc/samples/fld_lines.cpp index 62a2d48cb39..fdc148bd033 100644 --- a/modules/ximgproc/samples/fld_lines.cpp +++ b/modules/ximgproc/samples/fld_lines.cpp @@ -37,8 +37,9 @@ int main(int argc, char** argv) // hysteresis procedure in Canny() // canny_th2 50 - Second threshold for // hysteresis procedure in Canny() - // canny_aperture_size 3 - Aperturesize for the sobel - // operator in Canny() + // canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). + // If zero, Canny() is not applied and the input + // image is taken as an edge image. // do_merge false - If true, incremental merging of segments // will be perfomred int length_threshold = 10; diff --git a/modules/ximgproc/src/fast_line_detector.cpp b/modules/ximgproc/src/fast_line_detector.cpp index 33560184d2a..4217d9b33c5 100644 --- a/modules/ximgproc/src/fast_line_detector.cpp +++ b/modules/ximgproc/src/fast_line_detector.cpp @@ -29,10 +29,11 @@ class FastLineDetectorImpl : public FastLineDetector * _ hysteresis procedure in Canny() * @param _canny_th2 50 - Second threshold for * _ hysteresis procedure in Canny() - * @param _canny_aperture_size 3 - Aperturesize for the sobel - * _ operator in Canny() + * @param _canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). + * If zero, Canny() is not applied and the input + * image is taken as an edge image. * @param _do_merge false - If true, incremental merging of segments - will be perfomred + * will be performed */ FastLineDetectorImpl(int _length_threshold = 10, float _distance_threshold = 1.414213562f, double _canny_th1 = 50.0, double _canny_th2 = 50.0, int _canny_aperture_size = 3, @@ -80,7 +81,7 @@ class FastLineDetectorImpl : public FastLineDetector double distPointLine(const Mat& p, Mat& l); - void extractSegments(const std::vector& points, std::vector& segments ); + void extractSegments(const std::vector& points, std::vector& segments); void lineDetection(const Mat& src, std::vector& segments_all); @@ -113,7 +114,7 @@ FastLineDetectorImpl::FastLineDetectorImpl(int _length_threshold, float _distanc canny_th1(_canny_th1), canny_th2(_canny_th2), canny_aperture_size(_canny_aperture_size), do_merge(_do_merge) { CV_Assert(_length_threshold > 0 && _distance_threshold > 0 && - _canny_th1 > 0 && _canny_th2 > 0 && _canny_aperture_size > 0); + _canny_th1 > 0 && _canny_th2 > 0 && _canny_aperture_size >= 0); } void FastLineDetectorImpl::detect(InputArray _image, OutputArray _lines) @@ -344,7 +345,7 @@ template pt = T(pt_tmp); } -void FastLineDetectorImpl::extractSegments(const std::vector& points, std::vector& segments ) +void FastLineDetectorImpl::extractSegments(const std::vector& points, std::vector& segments) { bool is_line; @@ -544,8 +545,14 @@ void FastLineDetectorImpl::lineDetection(const Mat& src, std::vector& s std::vector points; std::vector segments, segments_tmp; Mat canny; - Canny(src, canny, canny_th1, canny_th2, canny_aperture_size); - + if (canny_aperture_size == 0) + { + canny = src; + } + else + { + Canny(src, canny, canny_th1, canny_th2, canny_aperture_size); + } canny.colRange(0,6).rowRange(0,6) = 0; canny.colRange(src.cols-5,src.cols).rowRange(src.rows-5,src.rows) = 0; diff --git a/modules/ximgproc/test/test_fld.cpp b/modules/ximgproc/test/test_fld.cpp index 80f877648bc..e4dfc9f4abd 100644 --- a/modules/ximgproc/test/test_fld.cpp +++ b/modules/ximgproc/test/test_fld.cpp @@ -23,6 +23,7 @@ class FLDBase : public testing::Test void GenerateWhiteNoise(Mat& image); void GenerateConstColor(Mat& image); void GenerateLines(Mat& image, const unsigned int numLines); + void GenerateEdgeLines(Mat& image, const unsigned int numLines); void GenerateBrokenLines(Mat& image, const unsigned int numLines); void GenerateRotatedRect(Mat& image); virtual void SetUp(); @@ -47,6 +48,7 @@ void FLDBase::GenerateConstColor(Mat& image) image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 256))); } + void FLDBase::GenerateLines(Mat& image, const unsigned int numLines) { image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 128))); @@ -60,6 +62,19 @@ void FLDBase::GenerateLines(Mat& image, const unsigned int numLines) } } +void FLDBase::GenerateEdgeLines(Mat& image, const unsigned int numLines) +{ + image = Mat(img_size, CV_8UC1, Scalar::all(0)); + + for(unsigned int i = 0; i < numLines; ++i) + { + int y = rng.uniform(10, img_size.width - 10); + Point p1(y, 10); + Point p2(y, img_size.height - 10); + line(image, p1, p2, Scalar(255), 1); + } +} + void FLDBase::GenerateBrokenLines(Mat& image, const unsigned int numLines) { image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 128))); @@ -145,6 +160,19 @@ TEST_F(ximgproc_FLD, lines) ASSERT_EQ(EPOCHS, passedtests); } +TEST_F(ximgproc_FLD, edgeLines) +{ + for (int i = 0; i < EPOCHS; ++i) + { + const unsigned int numOfLines = 1; + GenerateEdgeLines(test_image, numOfLines); + Ptr detector = createFastLineDetector(10, 1.414213562f, 50, 50, 0); + detector->detect(test_image, lines); + if(numOfLines == lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + TEST_F(ximgproc_FLD, mergeLines) { for (int i = 0; i < EPOCHS; ++i)