Merge pull request #1940 from tsenst:add_robust_optical_flow_implementation

Add robust local optical flow (RLOF) implementations (#1940)

* Add robust local optical flow (RLOF) implementations which is an improved pyramidal iterative Lucas-Kanade approach. This implementations contains interfaces for sparse optical flow for feature tracking and dense optical flow based on sparse-to-dense interpolation schemes.
Add performance and accuracy tests have been implementation as well as documentation with the related publications

* - exchange tabs with spaces
- fix optflow.bib indentation
- remove optflow_o.hpp
- change RLOFOpticalFlowParameter interfaces to Ptr<RLOFOpticalFlowParameter>
to remove error on building. Fix warnings

* introducing precompiler flag RLOD_SSE

* remove header that could not be found

* remove whitespaces
fix perf and accuracy tests

* remove x86intrin.h header

* fix ios and arm by removing last sse commands

* fix warnings for windows compilation

* fix documentation RLOFOpticalFlowParameter

* integrate cast to remove last warnings

* * add create method and function inferfaces to RLOFOpticalFlowParamter to enable python wrapper interfaces

* white space fixes / coding style

* fix perf test

* other changes: precomp.hpp / static

* use Matx44f and Vec4f instead of Mat

* normSigmas into constants

* replace ceil() calls

* maximum level is set to 5 so that it is similar value used in the papers

* implement paralellized horizontal cross segmentation as used in Geistert2016

* drop dead code

* Avoid using "data" and "step" calculations. Use .ptr<mat_type>(row, col) instead.

* Avoid using "data" and "step" calculations. Use .ptr<mat_type>(row, col) instead.

* bugfix on BEPLK with ica and adapt the accuracy tests

* more 'static' functions

* bugfix after changing ptr + step to .ptr(y,x) calls by adjusting ROI of
prevImage, currImage and derivI as well as changing the offset of the
points in the invoker classes.

* add some static_cast to avoid warning

* remove 50 grid size sample from perf test. This grid size is to sparse
for the epic interpolation

* remove notSameColor function since it is not used anymore

Author: tsenst

modules/optflow/CMakeLists.txt

@@ -1,2 +1,2 @@
 set(the_description "Optical Flow Algorithms")
-ocv_define_module(optflow opencv_core opencv_imgproc opencv_video opencv_ximgproc opencv_imgcodecs opencv_flann WRAP python)
+ocv_define_module(optflow opencv_core opencv_imgproc opencv_calib3d opencv_video opencv_ximgproc opencv_imgcodecs opencv_flann WRAP python)

modules/optflow/doc/optflow.bib

@@ -61,3 +61,52 @@ @inproceedings{Wang_2016_CVPR
   month = {June},
   year = {2016}
 }
+
+@inproceedings{Geistert2016,
+  author = {Jonas Geistert and Tobias Senst and Thomas Sikora},
+  title = {Robust Local Optical Flow: Dense Motion Vector Field Interpolation},
+  booktitle = {Picture Coding Symposium},
+  year = {2016},
+  pages = {1--5},
+}
+
+@inproceedings{Senst2016,
+  author = {Tobias Senst and Jonas Geistert and Thomas Sikora},
+  title = {Robust local optical flow: Long-range motions and varying illuminations},
+  booktitle = {IEEE International Conference on Image Processing},
+  year = {2016},
+  pages = {4478--4482},
+}
+
+@inproceedings{Senst2014,
+  author = {Tobias Senst and Thilo Borgmann and Ivo Keller and Thomas Sikora},
+  title = {Cross based Robust Local Optical Flow},
+  booktitle = {21th IEEE International Conference on Image Processing},
+  year = {2014},
+  pages = {1967--1971},
+}
+
+@inproceedings{Senst2013,
+  author = {Tobias Senst and Jonas Geistert and Ivo Keller and Thomas Sikora},
+  title = {Robust Local Optical Flow Estimation using Bilinear Equations for Sparse Motion Estimation},
+  booktitle = {20th IEEE International Conference on Image Processing},
+  year = {2013},
+  pages = {2499--2503},
+}
+
+@article{Senst2012,
+  author = {Tobias Senst and Volker Eiselein and Thomas Sikora},
+  title = {Robust Local Optical Flow for Feature Tracking},
+  journal = {IEEE Transactions on Circuits and Systems for Video Technology},
+  year = {2012},
+  pages = {1377--1387},
+  volume = {22},
+  number = {9},
+}
+
+@article{Tibshirani2008,
+  title={Fast computation of the median by successive binning},
+  author={Tibshirani, Ryan J},
+  journal={arXiv preprint arXiv:0806.3301},
+  year={2008}
+}

modules/optflow/include/opencv2/optflow.hpp

@@ -68,7 +68,7 @@ Functions reading and writing .flo files in "Middlebury" format, see: <http://vi
 
 #include "opencv2/optflow/pcaflow.hpp"
 #include "opencv2/optflow/sparse_matching_gpc.hpp"
-
+#include "opencv2/optflow/rlofflow.hpp"
 namespace cv
 {
 namespace optflow

modules/optflow/include/opencv2/optflow/rlofflow.hpp

@@ -0,0 +1,72 @@
+#include "perf_precomp.hpp"
+namespace opencv_test { namespace {
+
+typedef tuple<std::string, std::string, bool> ST_SR_IM_Sparse_t;
+typedef TestBaseWithParam<ST_SR_IM_Sparse_t> ST_SR_IM_Sparse;
+PERF_TEST_P(ST_SR_IM_Sparse, OpticalFlow_SparseRLOF,
+    testing::Combine(
+        testing::Values<std::string>("ST_BILINEAR", "ST_STANDART"),
+        testing::Values<std::string>("SR_CROSS", "SR_FIXED"),
+        testing::Values(true, false))
+)
+{
+    Mat frame1 = imread(getDataPath("cv/optflow/RubberWhale1.png"));
+    Mat frame2 = imread(getDataPath("cv/optflow/RubberWhale2.png"));
+    ASSERT_FALSE(frame1.empty());
+    ASSERT_FALSE(frame2.empty());
+    vector<Point2f> prevPts, currPts;
+    for (int r = 0; r < frame1.rows; r += 10)
+    {
+        for (int c = 0; c < frame1.cols; c += 10)
+        {
+            prevPts.push_back(Point2f(static_cast<float>(c), static_cast<float>(r)));
+        }
+    }
+    vector<uchar> status(prevPts.size());
+    vector<float> err(prevPts.size());
+
+    Ptr<RLOFOpticalFlowParameter> param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter);
+    if (get<0>(GetParam()) == "ST_BILINEAR")
+        param->solverType = ST_BILINEAR;
+    if (get<0>(GetParam()) == "ST_STANDART")
+        param->solverType = ST_STANDART;
+    if (get<1>(GetParam()) == "SR_CROSS")
+        param->supportRegionType = SR_CROSS;
+    if (get<1>(GetParam()) == "SR_FIXED")
+        param->supportRegionType = SR_FIXED;
+    param->useIlluminationModel = get<2>(GetParam());
+
+    PERF_SAMPLE_BEGIN()
+        calcOpticalFlowSparseRLOF(frame1, frame2, prevPts, currPts, status, err, param, 1.f);
+    PERF_SAMPLE_END()
+
+    SANITY_CHECK_NOTHING();
+}
+
+typedef tuple<std::string, int> INTERP_GRID_Dense_t;
+typedef TestBaseWithParam<INTERP_GRID_Dense_t> INTERP_GRID_Dense;
+PERF_TEST_P(INTERP_GRID_Dense, OpticalFlow_DenseRLOF,
+    testing::Combine(
+        testing::Values<std::string>("INTERP_EPIC", "INTERP_GEO"),
+        testing::Values<int>(4,10))
+)
+{
+    Mat flow;
+    Mat frame1 = imread(getDataPath("cv/optflow/RubberWhale1.png"));
+    Mat frame2 = imread(getDataPath("cv/optflow/RubberWhale1.png"));
+    ASSERT_FALSE(frame1.empty());
+    ASSERT_FALSE(frame2.empty());
+    Ptr<RLOFOpticalFlowParameter> param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter);;
+    Ptr< DenseRLOFOpticalFlow> algo = DenseRLOFOpticalFlow::create();
+    InterpolationType interp_type = INTERP_EPIC;
+    if (get<0>(GetParam()) == "INTERP_EPIC")
+        interp_type = INTERP_EPIC;
+    if (get<0>(GetParam()) == "INTERP_GEO")
+        interp_type = INTERP_GEO;
+    PERF_SAMPLE_BEGIN()
+        calcOpticalFlowDenseRLOF(frame1, frame2,flow, param, 1.0f, Size(get<1>(GetParam()), get<1>(GetParam())), interp_type);
+    PERF_SAMPLE_END()
+    SANITY_CHECK_NOTHING();
+}
+
+}} // namespace