From dbf2df0ef8114f8fce2fb45f1d9e7f74a9c8c35a Mon Sep 17 00:00:00 2001 From: David Starke Date: Fri, 22 Dec 2017 15:29:49 -0800 Subject: [PATCH 1/7] Track external memory usage for matrices --- src/BackgroundSubtractor.cc | 18 ++---- src/FaceRecognizer.cc | 4 +- src/ImgProc.cc | 25 ++------- src/LDAWrap.cc | 8 +-- src/Matrix.cc | 106 ++++++++++++++++++++---------------- src/Matrix.h | 2 + src/OpenCV.cc | 29 +++------- src/Stereo.cc | 15 +---- src/VideoCaptureWrap.cc | 5 +- 9 files changed, 85 insertions(+), 127 deletions(-) diff --git a/src/BackgroundSubtractor.cc b/src/BackgroundSubtractor.cc index cf952364..0afa37ae 100644 --- a/src/BackgroundSubtractor.cc +++ b/src/BackgroundSubtractor.cc @@ -255,10 +255,6 @@ NAN_METHOD(BackgroundSubtractorWrap::ApplyMOG) { try { - Local fgMask = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(fgMask); - cv::Mat mat; if (Buffer::HasInstance(info[0])) { uint8_t *buf = (uint8_t *) Buffer::Data(info[0]->ToObject()); @@ -287,7 +283,7 @@ NAN_METHOD(BackgroundSubtractorWrap::ApplyMOG) { #endif } - img->mat = _fgMask; + Local fgMask = Matrix::CreateWrappedFromMat(_fgMask); mat.release(); argv[0] = Nan::Null(); @@ -344,9 +340,7 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; - Local im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *imgout = Nan::ObjectWrap::Unwrap(im_to_return); - imgout->mat = _fgMask; + Local im_to_return = Matrix::CreateWrappedFromMat(_fgMask); Local argv[] = { Nan::Null() @@ -372,7 +366,6 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) { SETUP_FUNCTION(BackgroundSubtractorWrap); int callback_arg = -1; int numargs = info.Length(); - int success = 1; Local cb; @@ -407,10 +400,7 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) { } else { //synchronous - return the image try { - Local fgMask = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(fgMask); - + Local fgMask; cv::Mat mat; if (Buffer::HasInstance(info[0])) { uint8_t *buf = (uint8_t *) Buffer::Data(info[0]->ToObject()); @@ -436,7 +426,7 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) { #else self->subtractor->operator()(mat, _fgMask); #endif - img->mat = _fgMask; + fgMask = Matrix::CreateWrappedFromMat(_fgMask); } mat.release(); diff --git a/src/FaceRecognizer.cc b/src/FaceRecognizer.cc index 47b55838..e5518df1 100644 --- a/src/FaceRecognizer.cc +++ b/src/FaceRecognizer.cc @@ -448,9 +448,7 @@ NAN_METHOD(FaceRecognizerWrap::GetMat) { m = self->rec->getMat(key); #endif - Local im = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im); - img->mat = m; + Local im = Matrix::CreateWrappedFromMat(m); info.GetReturnValue().Set(im); } diff --git a/src/ImgProc.cc b/src/ImgProc.cc index 4dc4efb3..9e77444f 100644 --- a/src/ImgProc.cc +++ b/src/ImgProc.cc @@ -36,9 +36,7 @@ NAN_METHOD(ImgProc::DistanceTransform) { cv::distanceTransform(inputImage, outputImage, distType, 0); // Wrap the output image - Local outMatrixWrap = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *outMatrix = Nan::ObjectWrap::Unwrap(outMatrixWrap); - outMatrix->mat = outputImage; + Local outMatrixWrap = Matrix::CreateWrappedFromMat(outputImage); // Return the output image info.GetReturnValue().Set(outMatrixWrap); @@ -75,9 +73,7 @@ NAN_METHOD(ImgProc::Undistort) { cv::undistort(inputImage, outputImage, K, dist); // Wrap the output image - Local outMatrixWrap = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *outMatrix = Nan::ObjectWrap::Unwrap(outMatrixWrap); - outMatrix->mat = outputImage; + Local outMatrixWrap = Matrix::CreateWrappedFromMat(outputImage); // Return the output image info.GetReturnValue().Set(outMatrixWrap); @@ -128,13 +124,8 @@ NAN_METHOD(ImgProc::InitUndistortRectifyMap) { cv::initUndistortRectifyMap(K, dist, R, newK, imageSize, m1type, map1, map2); // Wrap the output maps - Local map1Wrap = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *map1Matrix = Nan::ObjectWrap::Unwrap(map1Wrap); - map1Matrix->mat = map1; - - Local map2Wrap = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *map2Matrix = Nan::ObjectWrap::Unwrap(map2Wrap); - map2Matrix->mat = map2; + Local map1Wrap = Matrix::CreateWrappedFromMat(map1); + Local map2Wrap = Matrix::CreateWrappedFromMat(map2); // Make a return object with the two maps Local ret = Nan::New(); @@ -181,9 +172,7 @@ NAN_METHOD(ImgProc::Remap) { cv::remap(inputImage, outputImage, map1, map2, interpolation); // Wrap the output image - Local outMatrixWrap = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *outMatrix = Nan::ObjectWrap::Unwrap(outMatrixWrap); - outMatrix->mat = outputImage; + Local outMatrixWrap = Matrix::CreateWrappedFromMat(outputImage); // Return the image info.GetReturnValue().Set(outMatrixWrap); @@ -223,9 +212,7 @@ NAN_METHOD(ImgProc::GetStructuringElement) { cv::Mat mat = cv::getStructuringElement(shape, ksize); // Wrap the output image - Local outMatrixWrap = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *outMatrix = ObjectWrap::Unwrap(outMatrixWrap); - outMatrix->mat = mat; + Local outMatrixWrap = Matrix::CreateWrappedFromMat(mat); // Return the image info.GetReturnValue().Set(outMatrixWrap); diff --git a/src/LDAWrap.cc b/src/LDAWrap.cc index 30867c8d..0dd2d6c5 100644 --- a/src/LDAWrap.cc +++ b/src/LDAWrap.cc @@ -66,9 +66,7 @@ NAN_METHOD(LDAWrap::SubspaceProject) { cv::Mat m = cv::subspaceProject(w->mat, mean->mat, src->mat); - Local im = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im); - img->mat = m; + Local im = Matrix::CreateWrappedFromMat(m); info.GetReturnValue().Set(im); } @@ -92,9 +90,7 @@ NAN_METHOD(LDAWrap::SubspaceReconstruct) { cv::Mat m = cv::subspaceReconstruct(w->mat, mean->mat, src->mat); - Local im = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im); - img->mat = m; + Local im = Matrix::CreateWrappedFromMat(m); info.GetReturnValue().Set(im); } diff --git a/src/Matrix.cc b/src/Matrix.cc index 01b4443d..d3a24e59 100755 --- a/src/Matrix.cc +++ b/src/Matrix.cc @@ -161,6 +161,16 @@ NAN_METHOD(Matrix::New) { info.GetReturnValue().Set(info.Holder()); } +//convenience factory method for creating a wrapped Matrix from a cv::Mat and tracking external memory correctly +Local Matrix::CreateWrappedFromMat(cv::Mat mat){ + Local < Object > result = + Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); + Matrix *m = Nan::ObjectWrap::Unwrap(result); + m->mat = mat; + Nan::AdjustExternalMemory(m->mat.rows * m->mat.cols * m->mat.elemSize()); + return result; +} + Matrix::Matrix() : node_opencv::Matrix() { mat = cv::Mat(); @@ -169,20 +179,24 @@ Matrix::Matrix() : Matrix::Matrix(int rows, int cols) : node_opencv::Matrix() { mat = cv::Mat(rows, cols, CV_32FC3); + Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); } Matrix::Matrix(int rows, int cols, int type) : node_opencv::Matrix() { mat = cv::Mat(rows, cols, type); + Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); } Matrix::Matrix(cv::Mat m, cv::Rect roi) : node_opencv::Matrix() { mat = cv::Mat(m, roi); + Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); } Matrix::Matrix(int rows, int cols, int type, Local scalarObj) { mat = cv::Mat(rows, cols, type); + Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); if (mat.channels() == 3) { mat.setTo(cv::Scalar(scalarObj->Get(0)->IntegerValue(), scalarObj->Get(1)->IntegerValue(), @@ -197,6 +211,11 @@ Matrix::Matrix(int rows, int cols, int type, Local scalarObj) { } } +Matrix::~Matrix(){ + int size = mat.rows * mat.cols * mat.elemSize(); + Nan::AdjustExternalMemory(-1 * size); +} + NAN_METHOD(Matrix::Empty) { SETUP_FUNCTION(Matrix) info.GetReturnValue().Set(Nan::New(self->mat.empty())); @@ -549,11 +568,7 @@ NAN_METHOD(Matrix::Type) { NAN_METHOD(Matrix::Clone) { SETUP_FUNCTION(Matrix) - Local < Object > im_h = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - - Matrix *m = Nan::ObjectWrap::Unwrap(im_h); - m->mat = self->mat.clone(); + Local im_h = Matrix::CreateWrappedFromMat(self->mat.clone()); info.GetReturnValue().Set(im_h); } @@ -571,10 +586,7 @@ NAN_METHOD(Matrix::Crop) { cv::Rect roi(x, y, width, height); - Local < Object > im_h = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m = Nan::ObjectWrap::Unwrap(im_h); - m->mat = self->mat(roi); + Local < Object > im_h = Matrix::CreateWrappedFromMat(self->mat(roi)); info.GetReturnValue().Set(im_h); } else { @@ -1115,11 +1127,8 @@ NAN_METHOD(Matrix::Zeros) { int h = info[1]->Uint32Value(); int type = (info.Length() > 2) ? info[2]->IntegerValue() : CV_64FC1; - Local im_h = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_h); cv::Mat mat = cv::Mat::zeros(w, h, type); - - img->mat = mat; + Local im_h = Matrix::CreateWrappedFromMat(mat); info.GetReturnValue().Set(im_h); } @@ -1130,11 +1139,9 @@ NAN_METHOD(Matrix::Ones) { int h = info[1]->Uint32Value(); int type = (info.Length() > 2) ? info[2]->IntegerValue() : CV_64FC1; - Local im_h = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_h); cv::Mat mat = cv::Mat::ones(w, h, type); + Local im_h = Matrix::CreateWrappedFromMat(mat); - img->mat = mat; info.GetReturnValue().Set(im_h); } @@ -1145,11 +1152,9 @@ NAN_METHOD(Matrix::Eye) { int h = info[1]->Uint32Value(); int type = (info.Length() > 2) ? info[2]->IntegerValue() : CV_64FC1; - Local im_h = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_h); cv::Mat mat = cv::Mat::eye(w, h, type); + Local im_h = Matrix::CreateWrappedFromMat(mat); - img->mat = mat; info.GetReturnValue().Set(im_h); } @@ -1161,7 +1166,10 @@ NAN_METHOD(Matrix::ConvertGrayscale) { Nan::ThrowError("Image is no 3-channel"); } + int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); cv::cvtColor(self->mat, self->mat, CV_BGR2GRAY); + int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + Nan::AdjustExternalMemory(newSize - oldSize); info.GetReturnValue().Set(Nan::Null()); } @@ -1292,6 +1300,7 @@ NAN_METHOD(Matrix::Sobel) { Matrix *result = Nan::ObjectWrap::Unwrap(result_to_return); cv::Sobel(self->mat, result->mat, ddepth, xorder, yorder, ksize, scale, delta, borderType); + Nan::AdjustExternalMemory(result->mat.rows * result->mat.cols * result->mat.elemSize()); info.GetReturnValue().Set(result_to_return); } @@ -1305,6 +1314,7 @@ NAN_METHOD(Matrix::Copy) { Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); self->mat.copyTo(img->mat); + Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); info.GetReturnValue().Set(img_to_return); } @@ -1324,6 +1334,7 @@ NAN_METHOD(Matrix::Flip) { Local img_to_return = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); cv::flip(self->mat, img->mat, flipCode); + Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); info.GetReturnValue().Set(img_to_return); } @@ -1338,8 +1349,6 @@ NAN_METHOD(Matrix::ROI) { } // Although it's an image to return, it is in fact a pointer to ROI of parent matrix - Local img_to_return = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); int x = info[0]->IntegerValue(); int y = info[1]->IntegerValue(); @@ -1347,7 +1356,8 @@ NAN_METHOD(Matrix::ROI) { int h = info[3]->IntegerValue(); cv::Mat roi(self->mat, cv::Rect(x,y,w,h)); - img->mat = roi; + // Although it's an image to return, it is in fact a pointer to ROI of parent matrix + Local img_to_return = Matrix::CreateWrappedFromMat(roi); info.GetReturnValue().Set(img_to_return); } @@ -1387,6 +1397,7 @@ NAN_METHOD(Matrix::Dct) { Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *m_out = Nan::ObjectWrap::Unwrap(out); m_out->mat.create(cols, rows, CV_32F); + Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); cv::dct(self->mat, m_out->mat); @@ -1403,6 +1414,7 @@ NAN_METHOD(Matrix::Idct) { Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *m_out = Nan::ObjectWrap::Unwrap(out); m_out->mat.create(cols, rows, CV_32F); + Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); cv::idct(self->mat, m_out->mat); @@ -1442,6 +1454,7 @@ NAN_METHOD(Matrix::Add) { Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *m_out = Nan::ObjectWrap::Unwrap(out); m_out->mat.create(cols, rows, self->mat.type()); + Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); try { cv::add(self->mat, src1->mat, m_out->mat); @@ -1937,9 +1950,7 @@ class ResizeASyncWorker: public Nan::AsyncWorker { if (success){ try{ - Local im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_to_return); - img->mat = dest->mat; + Local im_to_return = Matrix::CreateWrappedFromMat(dest->mat); delete dest; dest = NULL; @@ -2128,12 +2139,8 @@ NAN_METHOD(Matrix::GetRotationMatrix2D) { int y = Nan::To(info[2]).FromJust(); double scale = Nan::To(info[3]).FromMaybe(1.0); - Local img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - cv::Point center = cv::Point(x,y); - img->mat = getRotationMatrix2D(center, angle, scale); + Local img_to_return = Matrix::CreateWrappedFromMat(getRotationMatrix2D(center, angle, scale)); info.GetReturnValue().Set(img_to_return); } @@ -2298,6 +2305,7 @@ NAN_METHOD(Matrix::Threshold) { self->mat.copyTo(img->mat); cv::threshold(self->mat, img->mat, threshold, maxVal, typ); + Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); info.GetReturnValue().Set(img_to_return); } @@ -2318,6 +2326,7 @@ NAN_METHOD(Matrix::AdaptiveThreshold) { cv::adaptiveThreshold(self->mat, img->mat, maxVal, adaptiveMethod, thresholdType, blockSize, C); + Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); info.GetReturnValue().Set(img_to_return); } @@ -2333,6 +2342,8 @@ NAN_METHOD(Matrix::MeanStdDev) { Matrix *m_stddev = Nan::ObjectWrap::Unwrap(stddev); cv::meanStdDev(self->mat, m_mean->mat, m_stddev->mat); + Nan::AdjustExternalMemory(m_mean->mat.rows * m_mean->mat.cols * m_mean->mat.elemSize()); + Nan::AdjustExternalMemory(m_stddev->mat.rows * m_stddev->mat.cols * m_stddev->mat.elemSize()); Local data = Nan::New(); data->Set(Nan::New("mean").ToLocalChecked(), mean); @@ -2465,7 +2476,12 @@ NAN_METHOD(Matrix::CvtColor) { Nan::ThrowTypeError("Conversion code is unsupported"); } + int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); cv::cvtColor(self->mat, self->mat, iTransform); + int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + if(oldSize != newSize){ + Nan::AdjustExternalMemory(newSize - oldSize); + } return; } @@ -2489,10 +2505,7 @@ NAN_METHOD(Matrix::Split) { size = channels.size(); v8::Local arrChannels = Nan::New(size); for (unsigned int i = 0; i < size; i++) { - Local matObject = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix * m = Nan::ObjectWrap::Unwrap(matObject); - m->mat = channels[i]; + Local matObject = Matrix::CreateWrappedFromMat(channels[i]); arrChannels->Set(i, matObject); } @@ -2508,6 +2521,7 @@ NAN_METHOD(Matrix::Merge) { if (!info[0]->IsArray()) { Nan::ThrowTypeError("The argument must be an array"); } + int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); v8::Local jsChannels = v8::Local::Cast(info[0]); unsigned int L = jsChannels->Length(); @@ -2517,6 +2531,8 @@ NAN_METHOD(Matrix::Merge) { vChannels[i] = matObject->mat; } cv::merge(vChannels, self->mat); + int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + Nan::AdjustExternalMemory(newSize - oldSize); return; } @@ -2675,6 +2691,7 @@ NAN_METHOD(Matrix::MatchTemplateByMatrix) { int cols = self->mat.cols - templ->mat.cols + 1; int rows = self->mat.rows - templ->mat.rows + 1; m_out->mat.create(cols, rows, CV_32FC1); + Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); /* TM_SQDIFF =0 @@ -2709,6 +2726,7 @@ NAN_METHOD(Matrix::MatchTemplate) { int cols = self->mat.cols - templ.cols + 1; int rows = self->mat.rows - templ.rows + 1; m_out->mat.create(cols, rows, CV_32FC1); + Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); /* TM_SQDIFF =0 @@ -2862,9 +2880,7 @@ NAN_METHOD(Matrix::GetPerspectiveTransform) { tgt_corners[i] = cvPoint(tgtArray->Get(i*2)->IntegerValue(),tgtArray->Get(i*2+1)->IntegerValue()); } - Local xfrm = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *xfrmmat = Nan::ObjectWrap::Unwrap(xfrm); - xfrmmat->mat = cv::getPerspectiveTransform(src_corners, tgt_corners); + Local xfrm = Matrix::CreateWrappedFromMat(cv::getPerspectiveTransform(src_corners, tgt_corners)); info.GetReturnValue().Set(xfrm); } @@ -3019,11 +3035,7 @@ NAN_METHOD(Matrix::Reshape) { JSTHROW("Invalid number of arguments"); } - Local img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - - img->mat = self->mat.reshape(cn, rows); + Local img_to_return = Matrix::CreateWrappedFromMat(self->mat.reshape(cn, rows)); info.GetReturnValue().Set(img_to_return); } @@ -3032,7 +3044,9 @@ NAN_METHOD(Matrix::Release) { Nan::HandleScope scope; Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); + int size = self->mat.rows * self->mat.cols * self->mat.elemSize(); self->mat.release(); + Nan::AdjustExternalMemory(-1 * size); return; } @@ -3103,9 +3117,7 @@ NAN_METHOD(Matrix::Compare) { cv::Mat res = cv::Mat(width, height, CV_8UC1); cv::compare(self->mat, other->mat, res, cmpop); - Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_out = Nan::ObjectWrap::Unwrap(out); - m_out->mat = res; + Local out = Matrix::CreateWrappedFromMat(res); info.GetReturnValue().Set(out); return; @@ -3124,9 +3136,7 @@ NAN_METHOD(Matrix::Mul) { cv::Mat res = self->mat.mul(other->mat, scale); - Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_out = Nan::ObjectWrap::Unwrap(out); - m_out->mat = res; + Local out = Matrix::CreateWrappedFromMat(res); info.GetReturnValue().Set(out); return; diff --git a/src/Matrix.h b/src/Matrix.h index 81e89b21..9f69de97 100755 --- a/src/Matrix.h +++ b/src/Matrix.h @@ -7,11 +7,13 @@ class Matrix: public node_opencv::Matrix{ static Nan::Persistent constructor; static void Init(Local target); static NAN_METHOD(New); + static Local CreateWrappedFromMat(cv::Mat mat); Matrix(); Matrix(cv::Mat other, cv::Rect roi); Matrix(int rows, int cols); Matrix(int rows, int cols, int type); Matrix(int rows, int cols, int type, Local scalarObj); + ~Matrix(); static double DblGet(cv::Mat mat, int i, int j); diff --git a/src/OpenCV.cc b/src/OpenCV.cc index 6e8447dc..7ce38fe8 100755 --- a/src/OpenCV.cc +++ b/src/OpenCV.cc @@ -48,9 +48,7 @@ class AsyncImDecodeWorker: public Nan::AsyncWorker { Nan::HandleScope scope; try{ - Local im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_to_return); - img->mat = outputmat; + Local im_to_return = Matrix::CreateWrappedFromMat(outputmat); Local argv[] = { Nan::Null(), @@ -111,9 +109,7 @@ class AsyncImReadWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; - Local im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_to_return); - img->mat = outputmat; + Local im_to_return = Matrix::CreateWrappedFromMat(outputmat); Local argv[] = { Nan::Null(), @@ -169,10 +165,7 @@ NAN_METHOD(OpenCV::ReadImageAsync) { width = info[0]->Uint32Value(); height = info[1]->Uint32Value(); - Local img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - img->mat = *(new cv::Mat(width, height, type)); + Local img_to_return = Matrix::CreateWrappedFromMat(*(new cv::Mat(width, height, type))); if (callback_arg < 0){ info.GetReturnValue().Set(img_to_return); return; @@ -197,10 +190,7 @@ NAN_METHOD(OpenCV::ReadImageAsync) { } } if (callback_arg < 0){ - Local img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - img->mat = cv::imread(filename, flags); + Local img_to_return = Matrix::CreateWrappedFromMat(cv::imread(filename, flags)); info.GetReturnValue().Set(img_to_return); return; } else { @@ -222,13 +212,10 @@ NAN_METHOD(OpenCV::ReadImageAsync) { } } if (callback_arg < 0){ - Local img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); uint8_t *buf = (uint8_t *) Buffer::Data(info[0]->ToObject()); unsigned len = Buffer::Length(info[0]->ToObject()); cv::Mat *mbuf = new cv::Mat(len, 1, CV_64FC1, buf); - img->mat = cv::imdecode(*mbuf, flags); + Local img_to_return = Matrix::CreateWrappedFromMat(cv::imdecode(*mbuf, flags)); info.GetReturnValue().Set(img_to_return); return; } else { @@ -331,6 +318,7 @@ NAN_METHOD(OpenCV::ReadImage) { } img->mat = mat; + Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); } catch (cv::Exception& e) { argv[0] = Nan::Error(e.what()); argv[1] = Nan::Null(); @@ -383,10 +371,7 @@ NAN_METHOD(OpenCV::ReadImageMulti) { argv[1] = output; for (std::vector::size_type i = 0; i < mats.size(); i ++) { - Local im_h = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_h); - img->mat = mats[i]; - + Local im_h = Matrix::CreateWrappedFromMat(mats[i]); output->Set(i, im_h); } diff --git a/src/Stereo.cc b/src/Stereo.cc index c3af10f9..57e227d6 100644 --- a/src/Stereo.cc +++ b/src/Stereo.cc @@ -93,10 +93,7 @@ NAN_METHOD(StereoBM::Compute) { self->stereo(left, right, disparity, type); // Wrap the returned disparity map - Local < Object > disparityWrap = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *disp = Nan::ObjectWrap::Unwrap(disparityWrap); - disp->mat = disparity; + Local < Object > disparityWrap = Matrix::CreateWrappedFromMat(disparity); info.GetReturnValue().Set(disparityWrap); @@ -228,10 +225,7 @@ NAN_METHOD(StereoSGBM::Compute) { self->stereo(left, right, disparity); // Wrap the returned disparity map - Local < Object > disparityWrap = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *disp = Nan::ObjectWrap::Unwrap(disparityWrap); - disp->mat = disparity; + Local < Object > disparityWrap = Matrix::CreateWrappedFromMat(disparity); info.GetReturnValue().Set(disparityWrap); } catch (cv::Exception &e) { @@ -312,10 +306,7 @@ NAN_METHOD(StereoGC::Compute) { disp16.convertTo(disparity, CV_8U, -16); // Wrap the returned disparity map - Local < Object > disparityWrap = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *disp = Nan::ObjectWrap::Unwrap(disparityWrap); - disp->mat = disparity; + Local < Object > disparityWrap = Matrix::CreateWrappedFromMat(disparity); info.GetReturnValue().Set(disparityWrap); } catch (cv::Exception &e) { diff --git a/src/VideoCaptureWrap.cc b/src/VideoCaptureWrap.cc index d3e6c988..34e15d55 100755 --- a/src/VideoCaptureWrap.cc +++ b/src/VideoCaptureWrap.cc @@ -237,9 +237,7 @@ class AsyncVCWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; - Local im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_to_return); - img->mat = mat; + Local im_to_return = Matrix::CreateWrappedFromMat(mat); Local argv[] = { Nan::Null() @@ -280,6 +278,7 @@ NAN_METHOD(VideoCaptureWrap::ReadSync) { Matrix *img = Nan::ObjectWrap::Unwrap(im_to_return); v->cap.read(img->mat); + Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); info.GetReturnValue().Set(im_to_return); } From dff99718b92402fb38245e97e7082cce5d8e4834 Mon Sep 17 00:00:00 2001 From: David Starke Date: Thu, 4 Jan 2018 12:04:26 -0800 Subject: [PATCH 2/7] missed adjusting the external memory on resize --- src/Matrix.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Matrix.cc b/src/Matrix.cc index d3a24e59..04970010 100755 --- a/src/Matrix.cc +++ b/src/Matrix.cc @@ -2067,10 +2067,13 @@ NAN_METHOD(Matrix::Resize) { } else { try{ Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); + int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); cv::Mat res = cv::Mat(x, y, CV_32FC3); cv::resize(self->mat, res, cv::Size(x, y), 0, 0, interpolation); ~self->mat; self->mat = res; + int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + Nan::AdjustExternalMemory(newSize - oldSize); } catch (...){ return Nan::ThrowError("c++ Exception processing resize"); } From f8d7dc2f0631abd11c27570958f8c8e5aba0ed74 Mon Sep 17 00:00:00 2001 From: David Starke Date: Thu, 4 Jan 2018 20:16:36 -0800 Subject: [PATCH 3/7] =?UTF-8?q?More=20accurate=20tracking=20of=20external?= =?UTF-8?q?=20memory=20using=20OpenCV=E2=80=99s=20reference=20counts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/BackgroundSubtractor.cc | 23 +- src/Matrix.cc | 221 ++++---- src/Matrix.h | 3 + src/OpenCV.cc | 8 +- src/VideoCaptureWrap.cc | 10 +- test/memory.js | 980 ++++++++++++++++++++++++++++++++++++ 6 files changed, 1126 insertions(+), 119 deletions(-) create mode 100644 test/memory.js diff --git a/src/BackgroundSubtractor.cc b/src/BackgroundSubtractor.cc index 0afa37ae..15c1a0e1 100644 --- a/src/BackgroundSubtractor.cc +++ b/src/BackgroundSubtractor.cc @@ -283,7 +283,7 @@ NAN_METHOD(BackgroundSubtractorWrap::ApplyMOG) { #endif } - Local fgMask = Matrix::CreateWrappedFromMat(_fgMask); + Local fgMask = Matrix::CreateWrappedFromMat(_fgMask.clone()); mat.release(); argv[0] = Nan::Null(); @@ -309,15 +309,15 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker { AsyncBackgroundSubtractorWorker( Nan::Callback *callback, BackgroundSubtractorWrap *bg, - cv::Mat &img_mat): + Matrix *matrix): Nan::AsyncWorker(callback), bg(bg), - img_mat(img_mat) { // note: this makes a new cv::Mat, and so increments the ref count for the data without copying it + matrix(matrix) { } ~AsyncBackgroundSubtractorWorker() { - // upon destroy, img_mat will reduce refcount on data by one + } // Executed inside the worker-thread. @@ -328,9 +328,9 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker { // wait here if already in apply - auto-release on scope exit BGAutoMutex(bg->applymutex); #if CV_MAJOR_VERSION >= 3 - bg->subtractor->apply(this->img_mat, _fgMask); + bg->subtractor->apply(matrix->mat, _fgMask); #else - bg->subtractor->operator()(this->img_mat, _fgMask); + bg->subtractor->operator()(matrix->mat, _fgMask); #endif } @@ -340,7 +340,10 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; - Local im_to_return = Matrix::CreateWrappedFromMat(_fgMask); + delete matrix; + matrix = NULL; + + Local im_to_return = Matrix::CreateWrappedFromMat(_fgMask.clone()); Local argv[] = { Nan::Null() @@ -356,7 +359,7 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker { private: BackgroundSubtractorWrap *bg; - cv::Mat img_mat; + Matrix *matrix; cv::Mat _fgMask; }; @@ -395,7 +398,7 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) { Nan::Callback *callback = new Nan::Callback(cb.As()); Matrix *_img = Nan::ObjectWrap::Unwrap(info[0]->ToObject()); - Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, _img->mat)); + Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, new Matrix::Matrix(_img))); return; } else { //synchronous - return the image @@ -426,7 +429,7 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) { #else self->subtractor->operator()(mat, _fgMask); #endif - fgMask = Matrix::CreateWrappedFromMat(_fgMask); + fgMask = Matrix::CreateWrappedFromMat(_fgMask.clone()); } mat.release(); diff --git a/src/Matrix.cc b/src/Matrix.cc index 04970010..634cae45 100755 --- a/src/Matrix.cc +++ b/src/Matrix.cc @@ -161,13 +161,29 @@ NAN_METHOD(Matrix::New) { info.GetReturnValue().Set(info.Holder()); } -//convenience factory method for creating a wrapped Matrix from a cv::Mat and tracking external memory correctly +//Convenience factory method for creating a wrapped Matrix from a cv::Mat and tracking external memory correctly. +// Always tracks the referenced matrix as external memory. Local Matrix::CreateWrappedFromMat(cv::Mat mat){ Local < Object > result = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *m = Nan::ObjectWrap::Unwrap(result); m->mat = mat; - Nan::AdjustExternalMemory(m->mat.rows * m->mat.cols * m->mat.elemSize()); + Nan::AdjustExternalMemory(m->mat.dataend - m->mat.datastart); + + return result; +} + +//Convenience factory method for creating a wrapped Matrix from a cv::Mat and tracking external memory correctly. +// Only tracks the referenced matrix as external memory if the refcount does not exceed the base refcount. +// Useful for creating a wrapper Matrix around a Mat that is also referenced by another wrapper Matrix +Local Matrix::CreateWrappedFromMatIfNotReferenced(cv::Mat mat, int baseRefCount){ + Local < Object > result = + Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); + Matrix *m = Nan::ObjectWrap::Unwrap(result); + m->mat = mat; + if (m->getWrappedRefCount() <= 2 + baseRefCount){ //one reference in m, one on the stack + Nan::AdjustExternalMemory(m->mat.dataend - m->mat.datastart); + } return result; } @@ -179,24 +195,28 @@ Matrix::Matrix() : Matrix::Matrix(int rows, int cols) : node_opencv::Matrix() { mat = cv::Mat(rows, cols, CV_32FC3); - Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); + Nan::AdjustExternalMemory(mat.dataend - mat.datastart); } Matrix::Matrix(int rows, int cols, int type) : node_opencv::Matrix() { mat = cv::Mat(rows, cols, type); - Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); + Nan::AdjustExternalMemory(mat.dataend - mat.datastart); +} + +Matrix::Matrix(Matrix *m) : + node_opencv::Matrix() { + mat = cv::Mat(m->mat); } Matrix::Matrix(cv::Mat m, cv::Rect roi) : node_opencv::Matrix() { mat = cv::Mat(m, roi); - Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); } Matrix::Matrix(int rows, int cols, int type, Local scalarObj) { mat = cv::Mat(rows, cols, type); - Nan::AdjustExternalMemory(mat.rows * mat.cols * mat.elemSize()); + Nan::AdjustExternalMemory(mat.dataend - mat.datastart); if (mat.channels() == 3) { mat.setTo(cv::Scalar(scalarObj->Get(0)->IntegerValue(), scalarObj->Get(1)->IntegerValue(), @@ -212,8 +232,10 @@ Matrix::Matrix(int rows, int cols, int type, Local scalarObj) { } Matrix::~Matrix(){ - int size = mat.rows * mat.cols * mat.elemSize(); - Nan::AdjustExternalMemory(-1 * size); + if(getWrappedRefCount() == 1){ //if this holds the last reference to the Mat + int size = mat.dataend - mat.datastart; + Nan::AdjustExternalMemory(-1 * size); + } } NAN_METHOD(Matrix::Empty) { @@ -584,9 +606,9 @@ NAN_METHOD(Matrix::Crop) { int width = info[2]->IntegerValue(); int height = info[3]->IntegerValue(); - cv::Rect roi(x, y, width, height); + cv::Mat mat(self->mat, cv::Rect(x,y,width,height)); - Local < Object > im_h = Matrix::CreateWrappedFromMat(self->mat(roi)); + Local < Object > im_h = Matrix::CreateWrappedFromMatIfNotReferenced(mat, 1); info.GetReturnValue().Set(im_h); } else { @@ -784,10 +806,10 @@ NAN_METHOD(Matrix::ToBuffer) { class AsyncToBufferWorker: public Nan::AsyncWorker { public: - AsyncToBufferWorker(Nan::Callback *callback, cv::Mat mat, std::string ext, + AsyncToBufferWorker(Nan::Callback *callback, Matrix *matrix, std::string ext, std::vector params) : Nan::AsyncWorker(callback), - mat(mat), // dulipcate mat, adding ref, but not copying data + matrix(matrix), // dulipcate mat, adding ref, but not copying data ext(ext), params(params) { } @@ -799,13 +821,16 @@ class AsyncToBufferWorker: public Nan::AsyncWorker { void Execute() { std::vector vec(0); // std::vector params(0);//CV_IMWRITE_JPEG_QUALITY 90 - cv::imencode(ext, this->mat, vec, this->params); + cv::imencode(ext, matrix->mat, vec, this->params); res = vec; } void HandleOKCallback() { Nan::HandleScope scope; + delete matrix; + matrix = NULL; + Local buf = Nan::NewBuffer(res.size()).ToLocalChecked(); uchar* data = (uchar*) Buffer::Data(buf); memcpy(data, &res[0], res.size()); @@ -829,7 +854,7 @@ class AsyncToBufferWorker: public Nan::AsyncWorker { } private: - cv::Mat mat; + Matrix *matrix; std::string ext; std::vector params; std::vector res; @@ -869,7 +894,7 @@ NAN_METHOD(Matrix::ToBufferAsync) { } Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new AsyncToBufferWorker(callback, self->mat, ext, params)); + Nan::AsyncQueueWorker(new AsyncToBufferWorker(callback, new Matrix(self), ext, params)); return; } @@ -1062,9 +1087,9 @@ NAN_METHOD(Matrix::Save) { // https://github.com/rvagg/nan/blob/c579ae858ae3208d7e702e8400042ba9d48fa64b/examples/async_pi_estimate/async.cc class AsyncSaveWorker: public Nan::AsyncWorker { public: - AsyncSaveWorker(Nan::Callback *callback, cv::Mat mat, char* filename) : + AsyncSaveWorker(Nan::Callback *callback, Matrix *matrix, char* filename) : Nan::AsyncWorker(callback), - mat(mat), + matrix(matrix), filename(filename) { } @@ -1076,7 +1101,7 @@ class AsyncSaveWorker: public Nan::AsyncWorker { // here, so everything we need for input and output // should go on `this`. void Execute() { - res = cv::imwrite(this->filename, this->mat); + res = cv::imwrite(this->filename, matrix->mat); } // Executed when the async work is complete @@ -1085,6 +1110,9 @@ class AsyncSaveWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; + delete matrix; + matrix = NULL; + Local argv[] = { Nan::Null(), Nan::New(res) @@ -1098,7 +1126,7 @@ class AsyncSaveWorker: public Nan::AsyncWorker { } private: - cv::Mat mat; + Matrix *matrix; std::string filename; int res; }; @@ -1115,7 +1143,7 @@ NAN_METHOD(Matrix::SaveAsync) { REQ_FUN_ARG(1, cb); Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new AsyncSaveWorker(callback, self->mat, *filename)); + Nan::AsyncQueueWorker(new AsyncSaveWorker(callback, new Matrix(self), *filename)); return; } @@ -1166,9 +1194,9 @@ NAN_METHOD(Matrix::ConvertGrayscale) { Nan::ThrowError("Image is no 3-channel"); } - int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int oldSize = self->mat.dataend - self->mat.datastart; cv::cvtColor(self->mat, self->mat, CV_BGR2GRAY); - int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int newSize = self->mat.dataend - self->mat.datastart; Nan::AdjustExternalMemory(newSize - oldSize); info.GetReturnValue().Set(Nan::Null()); @@ -1300,7 +1328,7 @@ NAN_METHOD(Matrix::Sobel) { Matrix *result = Nan::ObjectWrap::Unwrap(result_to_return); cv::Sobel(self->mat, result->mat, ddepth, xorder, yorder, ksize, scale, delta, borderType); - Nan::AdjustExternalMemory(result->mat.rows * result->mat.cols * result->mat.elemSize()); + Nan::AdjustExternalMemory(result->mat.dataend - result->mat.datastart); info.GetReturnValue().Set(result_to_return); } @@ -1314,7 +1342,7 @@ NAN_METHOD(Matrix::Copy) { Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); self->mat.copyTo(img->mat); - Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); + Nan::AdjustExternalMemory(img->mat.dataend - img->mat.datastart); info.GetReturnValue().Set(img_to_return); } @@ -1334,7 +1362,7 @@ NAN_METHOD(Matrix::Flip) { Local img_to_return = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); cv::flip(self->mat, img->mat, flipCode); - Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); + Nan::AdjustExternalMemory(img->mat.dataend - img->mat.datastart); info.GetReturnValue().Set(img_to_return); } @@ -1357,7 +1385,7 @@ NAN_METHOD(Matrix::ROI) { cv::Mat roi(self->mat, cv::Rect(x,y,w,h)); // Although it's an image to return, it is in fact a pointer to ROI of parent matrix - Local img_to_return = Matrix::CreateWrappedFromMat(roi); + Local img_to_return = Matrix::CreateWrappedFromMatIfNotReferenced(roi, 1); info.GetReturnValue().Set(img_to_return); } @@ -1397,7 +1425,7 @@ NAN_METHOD(Matrix::Dct) { Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *m_out = Nan::ObjectWrap::Unwrap(out); m_out->mat.create(cols, rows, CV_32F); - Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); + Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); cv::dct(self->mat, m_out->mat); @@ -1414,7 +1442,7 @@ NAN_METHOD(Matrix::Idct) { Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *m_out = Nan::ObjectWrap::Unwrap(out); m_out->mat.create(cols, rows, CV_32F); - Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); + Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); cv::idct(self->mat, m_out->mat); @@ -1451,19 +1479,15 @@ NAN_METHOD(Matrix::Add) { Matrix *src1 = Nan::ObjectWrap::Unwrap(info[0]->ToObject()); - Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_out = Nan::ObjectWrap::Unwrap(out); - m_out->mat.create(cols, rows, self->mat.type()); - Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); - try { - cv::add(self->mat, src1->mat, m_out->mat); + cv::Mat outputmat = cv::Mat(cols, rows, self->mat.type()); + cv::add(self->mat, src1->mat, outputmat); + Local out = CreateWrappedFromMat(outputmat); + info.GetReturnValue().Set(out); } catch(cv::Exception& e ) { const char* err_msg = e.what(); Nan::ThrowError(err_msg); } - - info.GetReturnValue().Set(out); } NAN_METHOD(Matrix::BitwiseXor) { @@ -1916,10 +1940,10 @@ cv::Rect* setRect(Local objRect, cv::Rect &result) { class ResizeASyncWorker: public Nan::AsyncWorker { public: - ResizeASyncWorker(Nan::Callback *callback, cv::Mat image, cv::Size size, double fx, double fy, int interpolation) : + ResizeASyncWorker(Nan::Callback *callback, Matrix *image, cv::Size size, double fx, double fy, int interpolation) : Nan::AsyncWorker(callback), - image(image), // here, the cv::Mat is duplicated, adding to refcount without data copy - dest(NULL), + image(image), + dest(cv::Mat()), size(size), fx(fx), fy(fy), @@ -1928,17 +1952,14 @@ class ResizeASyncWorker: public Nan::AsyncWorker { } ~ResizeASyncWorker() { - // don't leave this if it was allocated - // could happen if NaN does not call HandleSuccess? - delete dest; - dest = NULL; - // cv::Mat image will be deleted, which will reduce refcount + // Any cleanup we needed to do could be done here. + // Clean up of the input image Matrix and the destination cv::Mat + // should be handled automatically by destructors. } void Execute() { try { - dest = new Matrix(); - cv::resize(image, dest->mat, size, fx, fy, interpolation); + cv::resize(image->mat, dest, size, fx, fy, interpolation); success = 1; } catch(...){ success = 0; @@ -1950,9 +1971,10 @@ class ResizeASyncWorker: public Nan::AsyncWorker { if (success){ try{ - Local im_to_return = Matrix::CreateWrappedFromMat(dest->mat); - delete dest; - dest = NULL; + Local im_to_return = Matrix::CreateWrappedFromMat(dest); + delete image; + image = NULL; + dest.release(); //release our refcount before handing it back to the callback Local argv[] = { Nan::Null(), // err @@ -1965,8 +1987,6 @@ class ResizeASyncWorker: public Nan::AsyncWorker { Nan::FatalException(try_catch); } } catch (...){ - delete dest; - dest = NULL; Local argv[] = { Nan::New("C++ exception wrapping response").ToLocalChecked(), // err Nan::Null() // result @@ -1979,9 +1999,6 @@ class ResizeASyncWorker: public Nan::AsyncWorker { } } } else { - delete dest; - dest = NULL; - Local argv[] = { Nan::New("C++ exception").ToLocalChecked(), // err Nan::Null() //result @@ -1996,8 +2013,8 @@ class ResizeASyncWorker: public Nan::AsyncWorker { } private: - cv::Mat image; - Matrix *dest; + Matrix *image; + cv::Mat dest; cv::Size size; double fx; double fy; @@ -2062,17 +2079,17 @@ NAN_METHOD(Matrix::Resize) { if (isAsync){ REQ_FUN_ARG(numargs-1, cb); Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new ResizeASyncWorker(callback, self->mat, size, fx, fy, interpolation)); + Nan::AsyncQueueWorker(new ResizeASyncWorker(callback, new Matrix(self), size, fx, fy, interpolation)); info.GetReturnValue().Set(Nan::Null()); } else { try{ Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); - int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int oldSize = (self->getWrappedRefCount() == 1) ? self->mat.dataend - self->mat.datastart : 0; cv::Mat res = cv::Mat(x, y, CV_32FC3); cv::resize(self->mat, res, cv::Size(x, y), 0, 0, interpolation); ~self->mat; self->mat = res; - int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int newSize = self->mat.dataend - self->mat.datastart; Nan::AdjustExternalMemory(newSize - oldSize); } catch (...){ return Nan::ThrowError("c++ Exception processing resize"); @@ -2302,13 +2319,12 @@ NAN_METHOD(Matrix::Threshold) { } } - Local < Object > img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - self->mat.copyTo(img->mat); + cv::Mat outputmat = cv::Mat(); + self->mat.copyTo(outputmat); + + cv::threshold(self->mat, outputmat, threshold, maxVal, typ); - cv::threshold(self->mat, img->mat, threshold, maxVal, typ); - Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); + Local < Object > img_to_return = CreateWrappedFromMat(outputmat); info.GetReturnValue().Set(img_to_return); } @@ -2322,14 +2338,13 @@ NAN_METHOD(Matrix::AdaptiveThreshold) { double blockSize = info[3]->NumberValue(); double C = info[4]->NumberValue(); - Local < Object > img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - self->mat.copyTo(img->mat); + cv::Mat outputmat = cv::Mat(); + self->mat.copyTo(outputmat); - cv::adaptiveThreshold(self->mat, img->mat, maxVal, adaptiveMethod, + cv::adaptiveThreshold(self->mat, outputmat, maxVal, adaptiveMethod, thresholdType, blockSize, C); - Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); + + Local < Object > img_to_return = CreateWrappedFromMat(outputmat); info.GetReturnValue().Set(img_to_return); } @@ -2339,18 +2354,14 @@ NAN_METHOD(Matrix::MeanStdDev) { Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); - Local mean = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_mean = Nan::ObjectWrap::Unwrap(mean); - Local stddev = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_stddev = Nan::ObjectWrap::Unwrap(stddev); + cv::Mat meanMat = cv::Mat(); + cv::Mat stddevMat = cv::Mat(); - cv::meanStdDev(self->mat, m_mean->mat, m_stddev->mat); - Nan::AdjustExternalMemory(m_mean->mat.rows * m_mean->mat.cols * m_mean->mat.elemSize()); - Nan::AdjustExternalMemory(m_stddev->mat.rows * m_stddev->mat.cols * m_stddev->mat.elemSize()); + cv::meanStdDev(self->mat, meanMat, stddevMat); Local data = Nan::New(); - data->Set(Nan::New("mean").ToLocalChecked(), mean); - data->Set(Nan::New("stddev").ToLocalChecked(), stddev); + data->Set(Nan::New("mean").ToLocalChecked(), CreateWrappedFromMat(meanMat)); + data->Set(Nan::New("stddev").ToLocalChecked(), CreateWrappedFromMat(stddevMat)); info.GetReturnValue().Set(data); } @@ -2479,9 +2490,9 @@ NAN_METHOD(Matrix::CvtColor) { Nan::ThrowTypeError("Conversion code is unsupported"); } - int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int oldSize = self->mat.dataend - self->mat.datastart; cv::cvtColor(self->mat, self->mat, iTransform); - int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int newSize = self->mat.dataend - self->mat.datastart; if(oldSize != newSize){ Nan::AdjustExternalMemory(newSize - oldSize); } @@ -2508,7 +2519,7 @@ NAN_METHOD(Matrix::Split) { size = channels.size(); v8::Local arrChannels = Nan::New(size); for (unsigned int i = 0; i < size; i++) { - Local matObject = Matrix::CreateWrappedFromMat(channels[i]); + Local matObject = Matrix::CreateWrappedFromMatIfNotReferenced(channels[i], 1); arrChannels->Set(i, matObject); } @@ -2524,7 +2535,7 @@ NAN_METHOD(Matrix::Merge) { if (!info[0]->IsArray()) { Nan::ThrowTypeError("The argument must be an array"); } - int oldSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int oldSize = self->mat.dataend - self->mat.datastart; v8::Local jsChannels = v8::Local::Cast(info[0]); unsigned int L = jsChannels->Length(); @@ -2534,7 +2545,7 @@ NAN_METHOD(Matrix::Merge) { vChannels[i] = matObject->mat; } cv::merge(vChannels, self->mat); - int newSize = self->mat.rows * self->mat.cols * self->mat.elemSize(); + int newSize = self->mat.dataend - self->mat.datastart; Nan::AdjustExternalMemory(newSize - oldSize); return; @@ -2694,7 +2705,7 @@ NAN_METHOD(Matrix::MatchTemplateByMatrix) { int cols = self->mat.cols - templ->mat.cols + 1; int rows = self->mat.rows - templ->mat.rows + 1; m_out->mat.create(cols, rows, CV_32FC1); - Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); + Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); /* TM_SQDIFF =0 @@ -2729,7 +2740,7 @@ NAN_METHOD(Matrix::MatchTemplate) { int cols = self->mat.cols - templ.cols + 1; int rows = self->mat.rows - templ.rows + 1; m_out->mat.create(cols, rows, CV_32FC1); - Nan::AdjustExternalMemory(m_out->mat.rows * m_out->mat.cols * m_out->mat.elemSize()); + Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); /* TM_SQDIFF =0 @@ -3038,7 +3049,7 @@ NAN_METHOD(Matrix::Reshape) { JSTHROW("Invalid number of arguments"); } - Local img_to_return = Matrix::CreateWrappedFromMat(self->mat.reshape(cn, rows)); + Local img_to_return = Matrix::CreateWrappedFromMatIfNotReferenced(self->mat.reshape(cn, rows),0); info.GetReturnValue().Set(img_to_return); } @@ -3047,9 +3058,13 @@ NAN_METHOD(Matrix::Release) { Nan::HandleScope scope; Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); - int size = self->mat.rows * self->mat.cols * self->mat.elemSize(); + + if(self->getWrappedRefCount() == 1){ + int size = self->mat.dataend - self->mat.datastart; + Nan::AdjustExternalMemory(-1 * size); + } + self->mat.release(); - Nan::AdjustExternalMemory(-1 * size); return; } @@ -3065,25 +3080,29 @@ NAN_METHOD(Matrix::Release) { //} -NAN_METHOD(Matrix::GetrefCount) { - Nan::HandleScope scope; - Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); - - int refcount = -1; - +int Matrix::getWrappedRefCount(){ + int refcount = -1; #if CV_MAJOR_VERSION >= 3 - if (self->mat.u){ - refcount = self->mat.u->refcount; + if (mat.u){ + refcount = mat.u->refcount; } else { refcount = -1; // indicates no reference ptr } #else - if (self->mat.refcount){ - refcount = *(self->mat.refcount); + if (mat.refcount){ + refcount = *(mat.refcount); } else { refcount = -1; // indicates no reference ptr } -#endif +#endif + return refcount; +} + +NAN_METHOD(Matrix::GetrefCount) { + Nan::HandleScope scope; + Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); + + int refcount = self->getWrappedRefCount(); info.GetReturnValue().Set(Nan::New(refcount)); return; diff --git a/src/Matrix.h b/src/Matrix.h index 9f69de97..59f8e211 100755 --- a/src/Matrix.h +++ b/src/Matrix.h @@ -8,7 +8,10 @@ class Matrix: public node_opencv::Matrix{ static void Init(Local target); static NAN_METHOD(New); static Local CreateWrappedFromMat(cv::Mat mat); + static Local CreateWrappedFromMatIfNotReferenced(cv::Mat mat, int baseRefCount); + int getWrappedRefCount(); Matrix(); + Matrix(Matrix *other); Matrix(cv::Mat other, cv::Rect roi); Matrix(int rows, int cols); Matrix(int rows, int cols, int type); diff --git a/src/OpenCV.cc b/src/OpenCV.cc index 7ce38fe8..27509961 100755 --- a/src/OpenCV.cc +++ b/src/OpenCV.cc @@ -49,6 +49,7 @@ class AsyncImDecodeWorker: public Nan::AsyncWorker { try{ Local im_to_return = Matrix::CreateWrappedFromMat(outputmat); + outputmat.release(); Local argv[] = { Nan::Null(), @@ -110,6 +111,7 @@ class AsyncImReadWorker: public Nan::AsyncWorker { Nan::HandleScope scope; Local im_to_return = Matrix::CreateWrappedFromMat(outputmat); + outputmat.release(); Local argv[] = { Nan::Null(), @@ -165,7 +167,7 @@ NAN_METHOD(OpenCV::ReadImageAsync) { width = info[0]->Uint32Value(); height = info[1]->Uint32Value(); - Local img_to_return = Matrix::CreateWrappedFromMat(*(new cv::Mat(width, height, type))); + Local img_to_return = Matrix::CreateWrappedFromMat(cv::Mat(width, height, type)); if (callback_arg < 0){ info.GetReturnValue().Set(img_to_return); return; @@ -287,7 +289,7 @@ NAN_METHOD(OpenCV::ReadImage) { } width = info[0]->Uint32Value(); height = info[1]->Uint32Value(); - mat = *(new cv::Mat(width, height, type)); + mat = cv::Mat(width, height, type); } else if (info[0]->IsString()) { std::string filename = std::string(*Nan::Utf8String(info[0]->ToString())); @@ -318,7 +320,7 @@ NAN_METHOD(OpenCV::ReadImage) { } img->mat = mat; - Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); + Nan::AdjustExternalMemory(img->mat.dataend - img->mat.datastart); } catch (cv::Exception& e) { argv[0] = Nan::Error(e.what()); argv[1] = Nan::Null(); diff --git a/src/VideoCaptureWrap.cc b/src/VideoCaptureWrap.cc index 34e15d55..0e1f4cc3 100755 --- a/src/VideoCaptureWrap.cc +++ b/src/VideoCaptureWrap.cc @@ -40,7 +40,7 @@ void VideoCaptureWrap::Init(Local target) { Nan::SetPrototypeMethod(ctor, "getFPS", GetFPS); Nan::SetPrototypeMethod(ctor, "setFPS", SetFPS); Nan::SetPrototypeMethod(ctor, "release", Release); - Nan::SetPrototypeMethod(ctor, "ReadSync", ReadSync); + Nan::SetPrototypeMethod(ctor, "readSync", ReadSync); Nan::SetPrototypeMethod(ctor, "grab", Grab); Nan::SetPrototypeMethod(ctor, "retrieve", Retrieve); @@ -238,6 +238,7 @@ class AsyncVCWorker: public Nan::AsyncWorker { Nan::HandleScope scope; Local im_to_return = Matrix::CreateWrappedFromMat(mat); + mat.release(); Local argv[] = { Nan::Null() @@ -274,11 +275,10 @@ NAN_METHOD(VideoCaptureWrap::ReadSync) { Nan::HandleScope scope; VideoCaptureWrap *v = Nan::ObjectWrap::Unwrap(info.This()); - Local im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_to_return); + cv::Mat outputmat = cv::Mat(); + v->cap.read(outputmat); - v->cap.read(img->mat); - Nan::AdjustExternalMemory(img->mat.rows * img->mat.cols * img->mat.elemSize()); + Local im_to_return = Matrix::CreateWrappedFromMat(outputmat); info.GetReturnValue().Set(im_to_return); } diff --git a/test/memory.js b/test/memory.js new file mode 100644 index 00000000..1d3abe40 --- /dev/null +++ b/test/memory.js @@ -0,0 +1,980 @@ +require("v8").setFlagsFromString('--expose_gc'); +var gc = require("vm").runInNewContext('gc'); + +var fs = require('fs') + , path = require('path') + , test = require('tape') + , cv = require('../lib/opencv'); + + var IMAGE_PATH = path.resolve(__dirname, '../examples/files', 'mona.png'); + var TEMP_SAVE_PATH = path.resolve(__dirname, '../examples/tmp', 'out.jpg'); + +// These tests check that every function that creates or modifies a Matrix handles its externally tracked memory correctly. +// Since the memory tracker uses OpenCV's reference counting to determine when to tell Node about memory changes, +// it is important that only Matrix objects that Javascript knows about retain references to internal OpenCV Mat objects. +// Reference counts for newly created objects should ususally therefore be 1, and releasing them should alter the +// externally tracked memory appropriately. + +// Note that garbage collection could run at any time, which could interfere with measurements of external memory, +// so these tests manually run garbage collection as part of their setup to minimize this possibility. + +//******************** +// Image Reading +//******************** + +test("readImage", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = cv.readImage(IMAGE_PATH); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 1134000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("readImage creating a new 100x100 matrix", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = cv.readImage(100,100); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 80000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("readImage from data buffer", t=>{ + gc(); + var buffer = fs.readFileSync(IMAGE_PATH); + var startingMemory = process.memoryUsage().external; + + var image = cv.readImage(buffer); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 1134000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("readImage (callback pattern)", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + cv.readImage(IMAGE_PATH, (err, image) => { + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 1134000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("readImage creating a new 100x100 matrix (callback pattern)", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + cv.readImage(100, 100, (err, image) => { + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 80000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("readImage from data buffer (callback pattern)", t=>{ + gc(); + var buffer = fs.readFileSync(IMAGE_PATH); + var startingMemory = process.memoryUsage().external; + + cv.readImage(buffer, (err, image) => { + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 1134000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("readImageAsync", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + cv.readImageAsync(IMAGE_PATH, (err, image) => { + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 1134000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("readImageAsync creating a new 100x100 matrix", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + cv.readImageAsync(100, 100, (err, image) => { + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 80000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("readImageAsync from data buffer", t=>{ + gc(); + var buffer = fs.readFileSync(IMAGE_PATH); + var startingMemory = process.memoryUsage().external; + + cv.readImageAsync(buffer, (err, image) => { + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 1134000); //image is tracked as external memory + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("video capture async", t=>{ + gc(); + var vid = new cv.VideoCapture(path.resolve(__dirname, '../examples/files', 'motion.mov')); + + var startingMemory = process.memoryUsage().external; + + vid.read( (err, im) => { + + t.equal(im.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 545280); //image is tracked as external memory + + im.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("video capture sync", t=>{ + gc(); + var vid = new cv.VideoCapture(path.resolve(__dirname, '../examples/files', 'motion.mov')); + + var startingMemory = process.memoryUsage().external; + + var im = vid.readSync(); + + t.equal(im.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 545280); + + im.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +//******************** +// Matrix Constructors +//******************** + +// Base constructor, doesn't actually allocate any memory +test("Matrix()", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(); + + t.equal(image.getrefCount(), -1); + t.equal(process.memoryUsage().external - startingMemory, 0); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +// Constructor with a size +test("Matrix(int, int)", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 120000); //100 * 100 * size of CV_32FC3 (12) + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +// Constructor with a size and type +test("Matrix(int, int, type)", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC1); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 10000); //100 * 100 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +// Constructor with a size, type, and initial values +test("Matrix(int, int, type, [values])", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + + +// Constructor with an existing matrix and a region of interest +test("Matrix(Matrix, x, y, w, h)", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + + t.equal(originalImage.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var image = new cv.Matrix(originalImage, 25, 25, 50, 50); //this should share memory with the original + t.equal(image.getrefCount(), 2); //so the refcount goes up + t.equal(process.memoryUsage().external - startingMemory, 30000); //but the memory usage does not + + originalImage.release(); + t.equal(originalImage.getrefCount(), -1); + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix.Zeros", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = cv.Matrix.Zeros(100, 100); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 80000); //100 * 100 * size of CV_64FC1 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix.Ones", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = cv.Matrix.Ones(100, 100); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 80000); //100 * 100 * size of CV_64FC1 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix.Eye", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = cv.Matrix.Eye(100, 100); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 80000); //100 * 100 * size of CV_64FC1 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +//******************** +// Matrix Functions +//******************** + +test("Matrix clone", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + + t.equal(originalImage.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var image = originalImage.clone(); + + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 60000); + + originalImage.release(); + t.equal(originalImage.getrefCount(), -1); + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix crop", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + + t.equal(originalImage.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + t.equal(originalImage.height(), 100); + + var image = originalImage.crop(25, 25, 50, 50); //crops share memory with the original + t.equal(originalImage.height(), 100); + t.equal(image.height(), 50); + t.equal(image.getrefCount(), 2); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + originalImage.release(); + t.equal(originalImage.getrefCount(), -1); + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +//ROI in this implementation is basically the same thing as crop +test("Matrix roi", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + + t.equal(originalImage.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + t.equal(originalImage.height(), 100); + + var image = originalImage.roi(25, 25, 50, 50); //ROIs share memory with the original + t.equal(originalImage.height(), 100); + t.equal(image.height(), 50); + t.equal(image.getrefCount(), 2); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + originalImage.release(); + t.equal(originalImage.getrefCount(), -1); + t.equal(image.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix convertGrayscale", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + originalImage.convertGrayscale(); + t.equal(process.memoryUsage().external - startingMemory, 10000); //grayscale takes less space + t.equal(originalImage.getrefCount(), 1); + + originalImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix sobel", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var resultImage = originalImage.sobel(cv.Constants.CV_16S, 1, 1); + + t.equal(process.memoryUsage().external - startingMemory, 90000); //our original 30k image plus our new 60k one + t.equal(originalImage.getrefCount(), 1); + t.equal(resultImage.getrefCount(), 1); + + originalImage.release(); + resultImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix copy", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var resultImage = originalImage.copy(0); + + t.equal(process.memoryUsage().external - startingMemory, 60000); //our original image plus our new one + t.equal(originalImage.getrefCount(), 1); + t.equal(resultImage.getrefCount(), 1); + + originalImage.release(); + resultImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix flip", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var resultImage = originalImage.flip(0); + + t.equal(process.memoryUsage().external - startingMemory, 60000); //our original image plus our new one + t.equal(originalImage.getrefCount(), 1); + t.equal(resultImage.getrefCount(), 1); + + originalImage.release(); + resultImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix dct", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_32F); + t.equal(process.memoryUsage().external - startingMemory, 40000); //100 * 100 * 4 + + var resultImage = originalImage.dct(); + + t.equal(process.memoryUsage().external - startingMemory, 80000); //our original image plus our new one + t.equal(originalImage.getrefCount(), 1); + t.equal(resultImage.getrefCount(), 1); + + originalImage.release(); + resultImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix idct", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_32F); + t.equal(process.memoryUsage().external - startingMemory, 40000); //100 * 100 * 4 + + var resultImage = originalImage.idct(); + + t.equal(process.memoryUsage().external - startingMemory, 80000); //our original image plus our new one + t.equal(originalImage.getrefCount(), 1); + t.equal(resultImage.getrefCount(), 1); + + originalImage.release(); + resultImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix add", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var src1 = new cv.Matrix.Ones(100, 100); + var src2 = new cv.Matrix.Ones(100, 100); + t.equal(process.memoryUsage().external - startingMemory, 160000); + + var resultMatrix = src1.add(src2); + + t.equal(resultMatrix.get(0,0), 2); //just making sure the result is correct + + t.equal(process.memoryUsage().external - startingMemory, 240000); + t.equal(src1.getrefCount(), 1); + t.equal(src2.getrefCount(), 1); + t.equal(resultMatrix.getrefCount(), 1); + + src1.release(); + src2.release(); + resultMatrix.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix resize", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + image.resize(50, 50); + t.equal(process.memoryUsage().external - startingMemory, 7500); //50 * 50 * 3 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix resize async", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + image.resize(50, 50, (err, resizedImage)=>{ + t.equal(image.height(), 100); //we have both the original and the resized image + t.equal(resizedImage.height(), 50); + + t.equal(image.getrefCount(), 1); + t.equal(resizedImage.getrefCount(), 1); + + + t.equal(process.memoryUsage().external - startingMemory, 37500); + + image.release(); + resizedImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); +}); + +test("Matrix resize async edge case", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(1000, 1000, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 3000000); //100 * 100 * 3 + + image.resize(50, 50, (err, resizedImage)=>{ + t.equal(image.getrefCount(), -1); //this happens second, image should have been released already + t.equal(resizedImage.getrefCount(), 1); + + t.equal(process.memoryUsage().external - startingMemory, 7500); + + resizedImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); + }); + image.release(); //this happens first +}); + +test("Matrix threshold", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8U); + t.equal(process.memoryUsage().external - startingMemory, 10000); //100 * 100 + + var resultImage = originalImage.threshold(1,1); + + t.equal(process.memoryUsage().external - startingMemory, 20000); //our original image plus our new one + t.equal(originalImage.getrefCount(), 1); + t.equal(resultImage.getrefCount(), 1); + + originalImage.release(); + resultImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix adaptiveThreshold", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var originalImage = new cv.Matrix(100, 100, cv.Constants.CV_8U); + t.equal(process.memoryUsage().external - startingMemory, 10000); //100 * 100 + + var resultImage = originalImage.adaptiveThreshold(255, 0, 0, 15, 2); + + t.equal(process.memoryUsage().external - startingMemory, 20000); //our original image plus our new one + t.equal(originalImage.getrefCount(), 1); + t.equal(resultImage.getrefCount(), 1); + + originalImage.release(); + resultImage.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix meanStdDev", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var matrix = new cv.Matrix.Ones(100, 100, cv.Constants.CV_8UC3); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + var result = matrix.meanStdDev(); + + t.equal(result.mean.getrefCount(), 1); + t.equal(result.stddev.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30048); + + matrix.release(); + result.mean.release(); + result.stddev.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix copyTo", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var smallImg = new cv.Matrix.Ones(25, 25, cv.Constants.CV_8UC3); + var bigImg = cv.Matrix.Zeros(100, 100, cv.Constants.CV_8UC3); + t.equal(process.memoryUsage().external - startingMemory, 30000 + 1875); + + smallImg.copyTo(bigImg, 0, 0); + + t.equal(smallImg.getrefCount(), 1); + t.equal(bigImg.getrefCount(), 1); + t.equal(process.memoryUsage().external - startingMemory, 30000 + 1875); + + smallImg.release(); + bigImg.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix cvtColor", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + image.cvtColor("CV_BGR2GRAY"); + t.equal(process.memoryUsage().external - startingMemory, 10000); //grayscale is smaller + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix split", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var result = image.split(); + t.equal(process.memoryUsage().external - startingMemory, 60000); + + image.release(); + t.equal(process.memoryUsage().external - startingMemory, 30000); + + result.forEach(m => m.release()); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix merge", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image1 = new cv.Matrix(100, 100, cv.Constants.CV_8UC1, [0]); + var image2 = new cv.Matrix(100, 100, cv.Constants.CV_8UC1, [0]); + var image3 = new cv.Matrix(100, 100, cv.Constants.CV_8UC1, [0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var result = new cv.Matrix(10,10); + t.equal(process.memoryUsage().external - startingMemory, 30000 + 1200); + + result.merge([image1, image2, image3]); + + t.equal(process.memoryUsage().external - startingMemory, 60000); + + result.release(); + image1.release(); + image2.release(); + image3.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix reshape", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + var result = image.reshape(2); + + t.equal(process.memoryUsage().external - startingMemory, 30000); //reshape does not copy data, so the allocated size hasn't changed + + image.release(); + + t.equal(process.memoryUsage().external - startingMemory, 30000); + + result.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +//******************** +// Additional Asynchronous Matrix Functions +//******************** + + +test("Matrix toBuffer async edge case", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(1000, 1000, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 3000000); //100 * 100 * 3 + + image.toBufferAsync((err, buffer)=>{ + t.equal(image.getrefCount(), -1); //this happens second, image should have been released already + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 33006); //the size of the buffer, which hasn't been released yet + t.end(); + }); + image.release(); +}); + +test("Matrix save async edge case", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(1000, 1000, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 3000000); //100 * 100 * 3 + + image.saveAsync(TEMP_SAVE_PATH, (err, buffer)=>{ + t.equal(image.getrefCount(), -1); //this happens second, image should have been released already + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + + fs.unlinkSync(TEMP_SAVE_PATH); + t.end(); + }); + image.release(); +}); + +//******************** +// Background Subtractors +//******************** + +function testSyncBackgroundSubtractor(t, subtractor){ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(1000, 1000, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 3000000); //100 * 100 * 3 + + var output = subtractor.apply(image); + t.equal(image.getrefCount(), 1); + t.equal(output.getrefCount(), 1); + + image.release(); + output.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + + t.end(); +} + +function testAsyncBackgroundSubtractor(t, subtractor){ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(1000, 1000, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 3000000); //100 * 100 * 3 + + subtractor.apply(image, (err, output) => { + t.equal(image.getrefCount(), 1); + t.equal(output.getrefCount(), 1); + + image.release(); + output.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + + t.end(); + }); +} + +function testAsyncBackgroundSubtractorEarlyRelease(t, subtractor){ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(1000, 1000, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 3000000); //100 * 100 * 3 + + subtractor.apply(image, (err, output) => { + t.equal(image.getrefCount(), -1); + t.equal(output.getrefCount(), 1); + + output.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + + t.end(); + }); + + image.release(); +} + +test("default background subtractor", t=>{ + testSyncBackgroundSubtractor(t, new cv.BackgroundSubtractor()); +}); + +test("MOG background subtractor", t=>{ + testSyncBackgroundSubtractor(t, cv.BackgroundSubtractor.createMOG()); +}); + +test("MOG2 background subtractor", t=>{ + testSyncBackgroundSubtractor(t, cv.BackgroundSubtractor.createMOG2()); +}); + +test("GMG background subtractor", t=>{ + testSyncBackgroundSubtractor(t, cv.BackgroundSubtractor.createGMG()); +}); + +test("default background subtractor async", t=>{ + testAsyncBackgroundSubtractor(t, new cv.BackgroundSubtractor()); +}); + +test("MOG background subtractor async", t=>{ + testAsyncBackgroundSubtractor(t, cv.BackgroundSubtractor.createMOG()); +}); + +test("MOG2 background subtractor async", t=>{ + testAsyncBackgroundSubtractor(t, cv.BackgroundSubtractor.createMOG2()); +}); + +test("GMG background subtractor async", t=>{ + testAsyncBackgroundSubtractor(t, cv.BackgroundSubtractor.createGMG()); +}); + +test("default background subtractor async early release", t=>{ + testAsyncBackgroundSubtractorEarlyRelease(t, new cv.BackgroundSubtractor()); +}); + +test("MOG background subtractor async early release", t=>{ + testAsyncBackgroundSubtractorEarlyRelease(t, cv.BackgroundSubtractor.createMOG()); +}); + +test("MOG2 background subtractor async early release", t=>{ + testAsyncBackgroundSubtractorEarlyRelease(t, cv.BackgroundSubtractor.createMOG2()); +}); + +test("GMG background subtractor async early release", t=>{ + testAsyncBackgroundSubtractorEarlyRelease(t, cv.BackgroundSubtractor.createGMG()); +}); From 2cd3bd4ff489535d94f6059fb1e853d7345b7dab Mon Sep 17 00:00:00 2001 From: David Starke Date: Fri, 5 Jan 2018 11:49:36 -0800 Subject: [PATCH 4/7] slightly more clear memory handling for async tasks --- src/BackgroundSubtractor.cc | 4 ++-- src/Matrix.cc | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/BackgroundSubtractor.cc b/src/BackgroundSubtractor.cc index 15c1a0e1..3bcbe158 100644 --- a/src/BackgroundSubtractor.cc +++ b/src/BackgroundSubtractor.cc @@ -312,7 +312,7 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker { Matrix *matrix): Nan::AsyncWorker(callback), bg(bg), - matrix(matrix) { + matrix(new Matrix::Matrix(matrix)) { } @@ -398,7 +398,7 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) { Nan::Callback *callback = new Nan::Callback(cb.As()); Matrix *_img = Nan::ObjectWrap::Unwrap(info[0]->ToObject()); - Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, new Matrix::Matrix(_img))); + Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, _img)); return; } else { //synchronous - return the image diff --git a/src/Matrix.cc b/src/Matrix.cc index 634cae45..56b5fd65 100755 --- a/src/Matrix.cc +++ b/src/Matrix.cc @@ -809,7 +809,7 @@ class AsyncToBufferWorker: public Nan::AsyncWorker { AsyncToBufferWorker(Nan::Callback *callback, Matrix *matrix, std::string ext, std::vector params) : Nan::AsyncWorker(callback), - matrix(matrix), // dulipcate mat, adding ref, but not copying data + matrix(new Matrix(matrix)), // dulipcate matrix, adding ref, but not copying data ext(ext), params(params) { } @@ -894,7 +894,7 @@ NAN_METHOD(Matrix::ToBufferAsync) { } Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new AsyncToBufferWorker(callback, new Matrix(self), ext, params)); + Nan::AsyncQueueWorker(new AsyncToBufferWorker(callback, self, ext, params)); return; } @@ -1089,7 +1089,7 @@ class AsyncSaveWorker: public Nan::AsyncWorker { public: AsyncSaveWorker(Nan::Callback *callback, Matrix *matrix, char* filename) : Nan::AsyncWorker(callback), - matrix(matrix), + matrix(new Matrix(matrix)), filename(filename) { } @@ -1143,7 +1143,7 @@ NAN_METHOD(Matrix::SaveAsync) { REQ_FUN_ARG(1, cb); Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new AsyncSaveWorker(callback, new Matrix(self), *filename)); + Nan::AsyncQueueWorker(new AsyncSaveWorker(callback, self, *filename)); return; } @@ -1942,7 +1942,7 @@ class ResizeASyncWorker: public Nan::AsyncWorker { public: ResizeASyncWorker(Nan::Callback *callback, Matrix *image, cv::Size size, double fx, double fy, int interpolation) : Nan::AsyncWorker(callback), - image(image), + image(new Matrix(image)), dest(cv::Mat()), size(size), fx(fx), @@ -1968,12 +1968,13 @@ class ResizeASyncWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; + + delete image; + image = NULL; if (success){ try{ Local im_to_return = Matrix::CreateWrappedFromMat(dest); - delete image; - image = NULL; dest.release(); //release our refcount before handing it back to the callback Local argv[] = { @@ -2079,7 +2080,7 @@ NAN_METHOD(Matrix::Resize) { if (isAsync){ REQ_FUN_ARG(numargs-1, cb); Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new ResizeASyncWorker(callback, new Matrix(self), size, fx, fy, interpolation)); + Nan::AsyncQueueWorker(new ResizeASyncWorker(callback, self, size, fx, fy, interpolation)); info.GetReturnValue().Set(Nan::Null()); } else { try{ From 45c88941676b3b442e5a38c3b29775c37f86fd90 Mon Sep 17 00:00:00 2001 From: David Starke Date: Thu, 11 Jan 2018 15:47:30 -0800 Subject: [PATCH 5/7] Additional memory tracking for matrices --- src/BackgroundSubtractor.cc | 4 +- src/Calib3D.cc | 47 +++++++++------------ src/CascadeClassifierWrap.cc | 6 ++- src/FaceRecognizer.cc | 47 ++++++++++++++++----- src/Features2d.cc | 27 +++++++----- src/VideoWriterWrap.cc | 13 +++--- test/memory.js | 82 ++++++++++++++++++++++++++++++++++++ 7 files changed, 168 insertions(+), 58 deletions(-) diff --git a/src/BackgroundSubtractor.cc b/src/BackgroundSubtractor.cc index 3bcbe158..4526fb97 100644 --- a/src/BackgroundSubtractor.cc +++ b/src/BackgroundSubtractor.cc @@ -309,10 +309,10 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker { AsyncBackgroundSubtractorWorker( Nan::Callback *callback, BackgroundSubtractorWrap *bg, - Matrix *matrix): + Matrix *matrix_in): Nan::AsyncWorker(callback), bg(bg), - matrix(new Matrix::Matrix(matrix)) { + matrix(new Matrix(matrix_in)) { } diff --git a/src/Calib3D.cc b/src/Calib3D.cc index e8e5ddb7..f783d9e3 100644 --- a/src/Calib3D.cc +++ b/src/Calib3D.cc @@ -3,15 +3,6 @@ #ifdef HAVE_OPENCV_CALIB3D -inline Local matrixFromMat(cv::Mat &input) { - Local matrixWrap = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *matrix = Nan::ObjectWrap::Unwrap(matrixWrap); - matrix->mat = input; - - return matrixWrap; -} - inline cv::Mat matFromMatrix(Local matrix) { Matrix* m = Nan::ObjectWrap::Unwrap(matrix->ToObject()); return m->mat; @@ -237,11 +228,11 @@ NAN_METHOD(Calib3D::CalibrateCamera) { ret->Set(Nan::New("reprojectionError").ToLocalChecked(), Nan::New(error)); // K - Local KMatrixWrap = matrixFromMat(K); + Local KMatrixWrap = Matrix::CreateWrappedFromMat(K); ret->Set(Nan::New("K").ToLocalChecked(), KMatrixWrap); // dist - Local distMatrixWrap = matrixFromMat(dist); + Local distMatrixWrap = Matrix::CreateWrappedFromMat(dist); ret->Set(Nan::New("distortion").ToLocalChecked(), distMatrixWrap); // Per frame R and t, skiping for now @@ -287,11 +278,11 @@ NAN_METHOD(Calib3D::SolvePnP) { Local ret = Nan::New(); // rvec - Local rMatrixWrap = matrixFromMat(rvec); + Local rMatrixWrap = Matrix::CreateWrappedFromMat(rvec); ret->Set(Nan::New("rvec").ToLocalChecked(), rMatrixWrap); // tvec - Local tMatrixWrap = matrixFromMat(tvec); + Local tMatrixWrap = Matrix::CreateWrappedFromMat(tvec); ret->Set(Nan::New("tvec").ToLocalChecked(), tMatrixWrap); // Return @@ -334,7 +325,7 @@ NAN_METHOD(Calib3D::GetOptimalNewCameraMatrix) { newImageSize); // Wrap the output K - Local KMatrixWrap = matrixFromMat(Kout); + Local KMatrixWrap = Matrix::CreateWrappedFromMat(Kout); // Return the new K matrix info.GetReturnValue().Set(KMatrixWrap); @@ -394,28 +385,28 @@ NAN_METHOD(Calib3D::StereoCalibrate) { // Make the output arguments // k1 - Local K1MatrixWrap = matrixFromMat(k1); + Local K1MatrixWrap = Matrix::CreateWrappedFromMat(k1); // d1 - Local d1MatrixWrap = matrixFromMat(d1); + Local d1MatrixWrap = Matrix::CreateWrappedFromMat(d1); // k2 - Local K2MatrixWrap = matrixFromMat(k2); + Local K2MatrixWrap = Matrix::CreateWrappedFromMat(k2); // d2 - Local d2MatrixWrap = matrixFromMat(d2); + Local d2MatrixWrap = Matrix::CreateWrappedFromMat(d2); // R - Local RMatrixWrap = matrixFromMat(R); + Local RMatrixWrap = Matrix::CreateWrappedFromMat(R); // t - Local tMatrixWrap = matrixFromMat(t); + Local tMatrixWrap = Matrix::CreateWrappedFromMat(t); // E - Local EMatrixWrap = matrixFromMat(E); + Local EMatrixWrap = Matrix::CreateWrappedFromMat(E); // F - Local FMatrixWrap = matrixFromMat(F); + Local FMatrixWrap = Matrix::CreateWrappedFromMat(F); // Add to return object ret->Set(Nan::New("K1").ToLocalChecked(), K1MatrixWrap); @@ -479,11 +470,11 @@ NAN_METHOD(Calib3D::StereoRectify) { // Make the return object Local ret = Nan::New(); - ret->Set(Nan::New("R1").ToLocalChecked(), matrixFromMat(R1)); - ret->Set(Nan::New("R2").ToLocalChecked(), matrixFromMat(R2)); - ret->Set(Nan::New("P1").ToLocalChecked(), matrixFromMat(P1)); - ret->Set(Nan::New("P2").ToLocalChecked(), matrixFromMat(P2)); - ret->Set(Nan::New("Q").ToLocalChecked(), matrixFromMat(Q)); + ret->Set(Nan::New("R1").ToLocalChecked(), Matrix::CreateWrappedFromMat(R1)); + ret->Set(Nan::New("R2").ToLocalChecked(), Matrix::CreateWrappedFromMat(R2)); + ret->Set(Nan::New("P1").ToLocalChecked(), Matrix::CreateWrappedFromMat(P1)); + ret->Set(Nan::New("P2").ToLocalChecked(), Matrix::CreateWrappedFromMat(P2)); + ret->Set(Nan::New("Q").ToLocalChecked(), Matrix::CreateWrappedFromMat(Q)); // Return the rectification parameters info.GetReturnValue().Set(ret); @@ -557,7 +548,7 @@ NAN_METHOD(Calib3D::ReprojectImageTo3D) { cv::reprojectImageTo3D(disparity, depthImage, Q); // Wrap the depth image - Local depthImageMatrix = matrixFromMat(depthImage); + Local depthImageMatrix = Matrix::CreateWrappedFromMat(depthImage); info.GetReturnValue().Set(depthImageMatrix); } catch (cv::Exception &e) { diff --git a/src/CascadeClassifierWrap.cc b/src/CascadeClassifierWrap.cc index 40c04a19..de002741 100755 --- a/src/CascadeClassifierWrap.cc +++ b/src/CascadeClassifierWrap.cc @@ -50,7 +50,7 @@ class AsyncDetectMultiScale: public Nan::AsyncWorker { Matrix* im, double scale, int neighbors, int minw, int minh) : Nan::AsyncWorker(callback), cc(cc), - im(im), + im(new Matrix(im)), //copy the matrix so we aren't affected if the original is released scale(scale), neighbors(neighbors), minw(minw), @@ -82,7 +82,9 @@ class AsyncDetectMultiScale: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; - // this->matrix->Unref(); + + delete im; + im = NULL; Local < Value > argv[2]; v8::Local < v8::Array > arr = Nan::New < v8::Array > (this->res.size()); diff --git a/src/FaceRecognizer.cc b/src/FaceRecognizer.cc index e5518df1..b638632b 100644 --- a/src/FaceRecognizer.cc +++ b/src/FaceRecognizer.cc @@ -27,6 +27,8 @@ namespace cv { #define FISHER 2 // Todo, move somewhere useful +// Note: References to the returned object here should not be retained past the end of the calling function. +// Otherwise, node might not keep track of external memory usage correctly. cv::Mat fromMatrixOrFilename(Local v) { cv::Mat im; if (v->IsString()) { @@ -40,6 +42,20 @@ cv::Mat fromMatrixOrFilename(Local v) { return im; } +// Note: Use this function when you might need to retain the returned object past the end of the calling function, +// such as in asynchronous methods +Matrix *CreateFromMatrixOrFilename(Local v) { + if (v->IsString()) { + Matrix *im = new Matrix(); + std::string filename = std::string(*Nan::Utf8String(v->ToString())); + im->mat = cv::imread(filename); + Nan::AdjustExternalMemory(im->mat.dataend - im->mat.datastart); + // std::cout<< im.size(); + } else { + return new Matrix(Nan::ObjectWrap::Unwrap(v->ToObject())); + } +} + Nan::Persistent FaceRecognizerWrap::constructor; void FaceRecognizerWrap::Init(Local target) { @@ -188,7 +204,7 @@ Local UnwrapTrainingData(Nan::NAN_METHOD_ARGS_TYPE info, } int label = valarr->Get(0)->Uint32Value(); - cv::Mat im = fromMatrixOrFilename(valarr->Get(1)); + cv::Mat im = fromMatrixOrFilename(valarr->Get(1)); //this is ok because we clone the image im = im.clone(); if (im.channels() == 3) { cv::cvtColor(im, im, CV_RGB2GRAY); @@ -295,7 +311,9 @@ NAN_METHOD(FaceRecognizerWrap::PredictSync) { cv::Mat im = fromMatrixOrFilename(info[0]); // TODO CHECK! if (im.channels() == 3) { - cv::cvtColor(im, im, CV_RGB2GRAY); + cv::Mat previous = im; + im = cv::Mat(); + cv::cvtColor(previous, im, CV_RGB2GRAY); } int predictedLabel = -1; @@ -322,10 +340,10 @@ NAN_METHOD(FaceRecognizerWrap::PredictSync) { class PredictASyncWorker: public Nan::AsyncWorker { public: - PredictASyncWorker(Nan::Callback *callback, cv::Ptr rec, cv::Mat im) : + PredictASyncWorker(Nan::Callback *callback, cv::Ptr rec, Matrix *matrix_in) : Nan::AsyncWorker(callback), rec(rec), - im(im) { + matrix(new Matrix(matrix_in)) { predictedLabel = -1; confidence = 0.0; } @@ -334,7 +352,7 @@ class PredictASyncWorker: public Nan::AsyncWorker { } void Execute() { - this->rec->predict(this->im, this->predictedLabel, this->confidence); + rec->predict(matrix->mat, predictedLabel, confidence); #if CV_MAJOR_VERSION >= 3 // Older versions of OpenCV3 incorrectly returned label=0 at // confidence=DBL_MAX instead of label=-1 on failure. This can be removed @@ -350,6 +368,9 @@ class PredictASyncWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; + delete matrix; + matrix = NULL; + v8::Local res = Nan::New(); res->Set(Nan::New("id").ToLocalChecked(), Nan::New(predictedLabel)); res->Set(Nan::New("confidence").ToLocalChecked(), Nan::New(confidence)); @@ -367,7 +388,7 @@ class PredictASyncWorker: public Nan::AsyncWorker { private: cv::Ptr rec; - cv::Mat im; + Matrix *matrix; int predictedLabel; double confidence; }; @@ -381,13 +402,19 @@ NAN_METHOD(FaceRecognizerWrap::Predict) { REQ_FUN_ARG(1, cb); - cv::Mat im = fromMatrixOrFilename(info[0]); - if (im.channels() == 3) { - cv::cvtColor(im, im, CV_RGB2GRAY); + Matrix *m = CreateFromMatrixOrFilename(info[0]); + if (m->mat.channels() == 3) { + Matrix *replacement = new Matrix(); + cv::cvtColor(m->mat, replacement->mat, CV_RGB2GRAY); + Nan::AdjustExternalMemory(replacement->mat.dataend - replacement->mat.datastart); + delete m; + m = replacement; } Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new PredictASyncWorker(callback, self->rec, im)); + Nan::AsyncQueueWorker(new PredictASyncWorker(callback, self->rec, m)); + + delete m; return; } diff --git a/src/Features2d.cc b/src/Features2d.cc index f2b75cb0..67e53295 100644 --- a/src/Features2d.cc +++ b/src/Features2d.cc @@ -16,10 +16,10 @@ void Features::Init(Local target) { class AsyncDetectSimilarity: public Nan::AsyncWorker { public: - AsyncDetectSimilarity(Nan::Callback *callback, cv::Mat image1, cv::Mat image2) : + AsyncDetectSimilarity(Nan::Callback *callback, Matrix *image1, Matrix *image2) : Nan::AsyncWorker(callback), - image1(image1), - image2(image2), + image1(new Matrix(image1)), + image2(new Matrix(image2)), dissimilarity(0) { } @@ -42,11 +42,11 @@ class AsyncDetectSimilarity: public Nan::AsyncWorker { std::vector keypoints1; std::vector keypoints2; - detector->detect(image1, keypoints1); - detector->detect(image2, keypoints2); + detector->detect(image1->mat, keypoints1); + detector->detect(image2->mat, keypoints2); - extractor->compute(image1, keypoints1, descriptors1); - extractor->compute(image2, keypoints2, descriptors2); + extractor->compute(image1->mat, keypoints1, descriptors1); + extractor->compute(image2->mat, keypoints2, descriptors2); matcher->match(descriptors1, descriptors2, matches); @@ -85,6 +85,11 @@ class AsyncDetectSimilarity: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; + delete image1; + delete image2; + image1 = NULL; + image2 = NULL; + Local argv[2]; argv[0] = Nan::Null(); @@ -94,8 +99,8 @@ class AsyncDetectSimilarity: public Nan::AsyncWorker { } private: - cv::Mat image1; - cv::Mat image2; + Matrix *image1; + Matrix *image2; double dissimilarity; }; @@ -104,8 +109,8 @@ NAN_METHOD(Features::Similarity) { REQ_FUN_ARG(2, cb); - cv::Mat image1 = Nan::ObjectWrap::Unwrap(info[0]->ToObject())->mat; - cv::Mat image2 = Nan::ObjectWrap::Unwrap(info[1]->ToObject())->mat; + Matrix *image1 = Nan::ObjectWrap::Unwrap(info[0]->ToObject()); + Matrix *image2 = Nan::ObjectWrap::Unwrap(info[1]->ToObject()); Nan::Callback *callback = new Nan::Callback(cb.As()); diff --git a/src/VideoWriterWrap.cc b/src/VideoWriterWrap.cc index 2a095fb4..32adb2d3 100755 --- a/src/VideoWriterWrap.cc +++ b/src/VideoWriterWrap.cc @@ -90,10 +90,10 @@ NAN_METHOD(VideoWriterWrap::Release) { class AsyncVWWorker: public Nan::AsyncWorker { public: - AsyncVWWorker(Nan::Callback *callback, VideoWriterWrap *vw, cv::Mat mat) : + AsyncVWWorker(Nan::Callback *callback, VideoWriterWrap *vw, Matrix *matrix_in) : Nan::AsyncWorker(callback), vw(vw), - mat(mat) { + matrix(new Matrix(matrix_in)) { } ~AsyncVWWorker() { @@ -104,7 +104,7 @@ class AsyncVWWorker: public Nan::AsyncWorker { // here, so everything we need for input and output // should go on `this`. void Execute() { - this->vw->writer.write(mat); + this->vw->writer.write(matrix->mat); } // Executed when the async work is complete @@ -113,6 +113,9 @@ class AsyncVWWorker: public Nan::AsyncWorker { void HandleOKCallback() { Nan::HandleScope scope; + delete matrix; + matrix = NULL; + Local argv[] = { Nan::Null() }; @@ -126,7 +129,7 @@ class AsyncVWWorker: public Nan::AsyncWorker { private: VideoWriterWrap *vw; - cv::Mat mat; + Matrix *matrix; }; NAN_METHOD(VideoWriterWrap::Write) { @@ -137,7 +140,7 @@ NAN_METHOD(VideoWriterWrap::Write) { REQ_FUN_ARG(1, cb); Nan::Callback *callback = new Nan::Callback(cb.As()); - Nan::AsyncQueueWorker(new AsyncVWWorker(callback, v, im->mat)); + Nan::AsyncQueueWorker(new AsyncVWWorker(callback, v, im)); return; } diff --git a/test/memory.js b/test/memory.js index 1d3abe40..e414bb91 100644 --- a/test/memory.js +++ b/test/memory.js @@ -7,7 +7,9 @@ var fs = require('fs') , cv = require('../lib/opencv'); var IMAGE_PATH = path.resolve(__dirname, '../examples/files', 'mona.png'); + var VIDEO_PATH = path.resolve(__dirname, '../examples/files', 'motion.mov'); var TEMP_SAVE_PATH = path.resolve(__dirname, '../examples/tmp', 'out.jpg'); + var TEMP_VIDEO_PATH = path.resolve(__dirname, '../examples/tmp', 'out.mp4'); // These tests check that every function that creates or modifies a Matrix handles its externally tracked memory correctly. // Since the memory tracker uses OpenCV's reference counting to determine when to tell Node about memory changes, @@ -978,3 +980,83 @@ test("MOG2 background subtractor async early release", t=>{ test("GMG background subtractor async early release", t=>{ testAsyncBackgroundSubtractorEarlyRelease(t, cv.BackgroundSubtractor.createGMG()); }); + +//******************** +// cascade classifier +//******************** + +test("cascade classifier async", t=> { + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.readImage(path.resolve(__dirname, '../examples/files', 'mona.png')); + + t.equal(process.memoryUsage().external - startingMemory, 1134000); //100 * 100 * 3 + + var classifier = new cv.CascadeClassifier(cv.FACE_CASCADE); + + classifier.detectMultiScale(image, (err, faces) =>{ + t.equal(image.getrefCount(), 1); + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + + t.end(); + }); +}); + +test("cascade classifier async early release", t=> { + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.readImage(path.resolve(__dirname, '../examples/files', 'mona.png')); + + t.equal(process.memoryUsage().external - startingMemory, 1134000); + + var classifier = new cv.CascadeClassifier(cv.FACE_CASCADE); + + classifier.detectMultiScale(image, (err, faces) =>{ + t.equal(image.getrefCount(), -1); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + + t.end(); + }); + image.release(); +}); + +//******************** +// VideoWriter +//******************** + + +test("Video writer async edge case", t=>{ + gc(); + + var startingMemory = process.memoryUsage().external; + + var reader = new cv.VideoCapture(VIDEO_PATH); + + reader.read((err, image)=>{ + t.equal(process.memoryUsage().external - startingMemory, 545280); + var writer = new cv.VideoWriter(TEMP_VIDEO_PATH, 'mp4v', 1, image.size(), true); + + writer.write(image, err=>{ + t.equal(image.getrefCount(), -1); //this happens second, image should have been released already + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + + writer.release(); + fs.unlinkSync(TEMP_VIDEO_PATH); + t.end(); + }); + + image.release(); + }); +}); + +//******************** +// Face Recognizer +//******************** From 9646626e3d39b0be8c70b133f7ab3eae94a7892d Mon Sep 17 00:00:00 2001 From: David Starke Date: Wed, 17 Jan 2018 14:25:43 -0800 Subject: [PATCH 6/7] Consolidate external memory adjustment in Matrix --- src/FaceRecognizer.cc | 12 ++--- src/Matrix.cc | 108 ++++++++++++++++++------------------------ src/Matrix.h | 1 + src/OpenCV.cc | 13 ++--- 4 files changed, 56 insertions(+), 78 deletions(-) diff --git a/src/FaceRecognizer.cc b/src/FaceRecognizer.cc index b638632b..5a0e0345 100644 --- a/src/FaceRecognizer.cc +++ b/src/FaceRecognizer.cc @@ -48,8 +48,8 @@ Matrix *CreateFromMatrixOrFilename(Local v) { if (v->IsString()) { Matrix *im = new Matrix(); std::string filename = std::string(*Nan::Utf8String(v->ToString())); - im->mat = cv::imread(filename); - Nan::AdjustExternalMemory(im->mat.dataend - im->mat.datastart); + im->setMat(cv::imread(filename)); + return im; // std::cout<< im.size(); } else { return new Matrix(Nan::ObjectWrap::Unwrap(v->ToObject())); @@ -404,11 +404,9 @@ NAN_METHOD(FaceRecognizerWrap::Predict) { Matrix *m = CreateFromMatrixOrFilename(info[0]); if (m->mat.channels() == 3) { - Matrix *replacement = new Matrix(); - cv::cvtColor(m->mat, replacement->mat, CV_RGB2GRAY); - Nan::AdjustExternalMemory(replacement->mat.dataend - replacement->mat.datastart); - delete m; - m = replacement; + cv::Mat grayMat; + cv::cvtColor(m->mat, grayMat, CV_RGB2GRAY); + m->setMat(grayMat); } Nan::Callback *callback = new Nan::Callback(cb.As()); diff --git a/src/Matrix.cc b/src/Matrix.cc index 56b5fd65..1e6f2923 100755 --- a/src/Matrix.cc +++ b/src/Matrix.cc @@ -167,8 +167,7 @@ Local Matrix::CreateWrappedFromMat(cv::Mat mat){ Local < Object > result = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); Matrix *m = Nan::ObjectWrap::Unwrap(result); - m->mat = mat; - Nan::AdjustExternalMemory(m->mat.dataend - m->mat.datastart); + m->setMat(mat); return result; } @@ -238,6 +237,20 @@ Matrix::~Matrix(){ } } +// Set the wrapped Mat with correct memory tracking +// For this to work correctly, there should be no external references held to our previous +// wrapped Mat, and no other Matrix objects should wrap a Mat pointing at the same +// memory as our new Mat. +void Matrix::setMat(cv::Mat m){ + int oldSize = 0; + if(getWrappedRefCount() == 1){ //if this holds the last reference to the Mat + oldSize = mat.dataend - mat.datastart; + } + mat = m; + int newSize = mat.dataend - mat.datastart; + Nan::AdjustExternalMemory(newSize - oldSize); +} + NAN_METHOD(Matrix::Empty) { SETUP_FUNCTION(Matrix) info.GetReturnValue().Set(Nan::New(self->mat.empty())); @@ -1323,14 +1336,10 @@ NAN_METHOD(Matrix::Sobel) { Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); - Local result_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *result = Nan::ObjectWrap::Unwrap(result_to_return); - - cv::Sobel(self->mat, result->mat, ddepth, xorder, yorder, ksize, scale, delta, borderType); - Nan::AdjustExternalMemory(result->mat.dataend - result->mat.datastart); + cv::Mat result; + cv::Sobel(self->mat, result, ddepth, xorder, yorder, ksize, scale, delta, borderType); - info.GetReturnValue().Set(result_to_return); + info.GetReturnValue().Set(CreateWrappedFromMat(result)); } NAN_METHOD(Matrix::Copy) { @@ -1338,13 +1347,10 @@ NAN_METHOD(Matrix::Copy) { Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); - Local img_to_return = - Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - self->mat.copyTo(img->mat); - Nan::AdjustExternalMemory(img->mat.dataend - img->mat.datastart); + cv::Mat result; + self->mat.copyTo(result); - info.GetReturnValue().Set(img_to_return); + info.GetReturnValue().Set(CreateWrappedFromMat(result)); } NAN_METHOD(Matrix::Flip) { @@ -1359,12 +1365,10 @@ NAN_METHOD(Matrix::Flip) { int flipCode = Nan::To(info[0]).FromJust(); - Local img_to_return = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(img_to_return); - cv::flip(self->mat, img->mat, flipCode); - Nan::AdjustExternalMemory(img->mat.dataend - img->mat.datastart); + cv::Mat result; + cv::flip(self->mat, result, flipCode); - info.GetReturnValue().Set(img_to_return); + info.GetReturnValue().Set(CreateWrappedFromMat(result)); } NAN_METHOD(Matrix::ROI) { @@ -1422,14 +1426,11 @@ NAN_METHOD(Matrix::Dct) { int cols = self->mat.cols; int rows = self->mat.rows; - Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_out = Nan::ObjectWrap::Unwrap(out); - m_out->mat.create(cols, rows, CV_32F); - Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); + cv::Mat result(cols, rows, CV_32F); - cv::dct(self->mat, m_out->mat); + cv::dct(self->mat, result); - info.GetReturnValue().Set(out); + info.GetReturnValue().Set(CreateWrappedFromMat(result)); } NAN_METHOD(Matrix::Idct) { @@ -1439,14 +1440,11 @@ NAN_METHOD(Matrix::Idct) { int cols = self->mat.cols; int rows = self->mat.rows; - Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_out = Nan::ObjectWrap::Unwrap(out); - m_out->mat.create(cols, rows, CV_32F); - Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); + cv::Mat result(cols, rows, CV_32F); - cv::idct(self->mat, m_out->mat); + cv::idct(self->mat, result); - info.GetReturnValue().Set(out); + info.GetReturnValue().Set(CreateWrappedFromMat(result)); } NAN_METHOD(Matrix::AddWeighted) { @@ -2085,13 +2083,9 @@ NAN_METHOD(Matrix::Resize) { } else { try{ Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); - int oldSize = (self->getWrappedRefCount() == 1) ? self->mat.dataend - self->mat.datastart : 0; cv::Mat res = cv::Mat(x, y, CV_32FC3); cv::resize(self->mat, res, cv::Size(x, y), 0, 0, interpolation); - ~self->mat; - self->mat = res; - int newSize = self->mat.dataend - self->mat.datastart; - Nan::AdjustExternalMemory(newSize - oldSize); + self->setMat(res); } catch (...){ return Nan::ThrowError("c++ Exception processing resize"); } @@ -2120,8 +2114,7 @@ NAN_METHOD(Matrix::Rotate) { // See if we do right angle rotation, we transpose the matrix: if (angle2 % 180) { cv::transpose(self->mat, res); - ~self->mat; - self->mat = res; + self->setMat(res); } // Now flip the image int mode = -1;// flip around both axes @@ -2143,8 +2136,7 @@ NAN_METHOD(Matrix::Rotate) { rotMatrix = getRotationMatrix2D(center, angle, 1.0); cv::warpAffine(self->mat, res, rotMatrix, self->mat.size()); - ~self->mat; - self->mat = res; + self->setMat(res); return; } @@ -2180,8 +2172,7 @@ NAN_METHOD(Matrix::WarpAffine) { cv::Size resSize = cv::Size(dstRows, dstCols); cv::warpAffine(self->mat, res, rotMatrix->mat, resSize); - ~self->mat; - self->mat = res; + self->setMat(res); return; } @@ -2701,12 +2692,9 @@ NAN_METHOD(Matrix::MatchTemplateByMatrix) { Matrix *self = Nan::ObjectWrap::Unwrap(info.This()); Matrix *templ = Nan::ObjectWrap::Unwrap(info[0]->ToObject()); - Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_out = Nan::ObjectWrap::Unwrap(out); int cols = self->mat.cols - templ->mat.cols + 1; int rows = self->mat.rows - templ->mat.rows + 1; - m_out->mat.create(cols, rows, CV_32FC1); - Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); + cv::Mat result(cols, rows, CV_32FC1); /* TM_SQDIFF =0 @@ -2719,8 +2707,8 @@ NAN_METHOD(Matrix::MatchTemplateByMatrix) { int method = (info.Length() < 2) ? (int)cv::TM_CCORR_NORMED : info[1]->Uint32Value(); if (!(method >= 0 && method <= 5)) method = (int)cv::TM_CCORR_NORMED; - cv::matchTemplate(self->mat, templ->mat, m_out->mat, method); - info.GetReturnValue().Set(out); + cv::matchTemplate(self->mat, templ->mat, result, method); + info.GetReturnValue().Set(CreateWrappedFromMat(result)); } // @author ytham @@ -2736,12 +2724,9 @@ NAN_METHOD(Matrix::MatchTemplate) { cv::Mat templ; templ = cv::imread(filename, -1); - Local out = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *m_out = Nan::ObjectWrap::Unwrap(out); int cols = self->mat.cols - templ.cols + 1; int rows = self->mat.rows - templ.rows + 1; - m_out->mat.create(cols, rows, CV_32FC1); - Nan::AdjustExternalMemory(m_out->mat.dataend - m_out->mat.datastart); + cv::Mat result(cols, rows, CV_32FC1); /* TM_SQDIFF =0 @@ -2753,15 +2738,15 @@ NAN_METHOD(Matrix::MatchTemplate) { */ int method = (info.Length() < 2) ? (int)cv::TM_CCORR_NORMED : info[1]->Uint32Value(); - cv::matchTemplate(self->mat, templ, m_out->mat, method); - cv::normalize(m_out->mat, m_out->mat, 0, 1, cv::NORM_MINMAX, -1, cv::Mat()); + cv::matchTemplate(self->mat, templ, result, method); + cv::normalize(result, result, 0, 1, cv::NORM_MINMAX, -1, cv::Mat()); double minVal; double maxVal; cv::Point minLoc; cv::Point maxLoc; cv::Point matchLoc; - minMaxLoc(m_out->mat, &minVal, &maxVal, &minLoc, &maxLoc, cv::Mat()); + minMaxLoc(result, &minVal, &maxVal, &minLoc, &maxLoc, cv::Mat()); if(method == CV_TM_SQDIFF || method == CV_TM_SQDIFF_NORMED) { matchLoc = minLoc; @@ -2782,10 +2767,10 @@ NAN_METHOD(Matrix::MatchTemplate) { cv::rectangle(self->mat, roi, cv::Scalar(0,0,255)); } - m_out->mat.convertTo(m_out->mat, CV_8UC1, 255, 0); + result.convertTo(result, CV_8UC1, 255, 0); v8::Local arr = Nan::New(5); - arr->Set(0, out); + arr->Set(0, CreateWrappedFromMat(result)); arr->Set(1, Nan::New(roi_x)); arr->Set(2, Nan::New(roi_y)); arr->Set(3, Nan::New(roi_width)); @@ -2923,8 +2908,7 @@ NAN_METHOD(Matrix::WarpPerspective) { cv::warpPerspective(self->mat, res, xfrm->mat, cv::Size(width, height), flags, borderMode, borderColor); - ~self->mat; - self->mat = res; + self->setMat(res); info.GetReturnValue().Set(Nan::Null()); } @@ -3025,8 +3009,8 @@ NAN_METHOD(Matrix::Shift) { cv::Rect roi = cv::Rect(std::max(-deltai.x, 0), std::max(-deltai.y, 0), 0, 0) + self->mat.size(); res = padded(roi); - ~self->mat; - self->mat = res; + + self->setMat(res); return; } diff --git a/src/Matrix.h b/src/Matrix.h index 59f8e211..336b9fcb 100755 --- a/src/Matrix.h +++ b/src/Matrix.h @@ -10,6 +10,7 @@ class Matrix: public node_opencv::Matrix{ static Local CreateWrappedFromMat(cv::Mat mat); static Local CreateWrappedFromMatIfNotReferenced(cv::Mat mat, int baseRefCount); int getWrappedRefCount(); + void setMat(cv::Mat mat); Matrix(); Matrix(Matrix *other); Matrix(cv::Mat other, cv::Rect roi); diff --git a/src/OpenCV.cc b/src/OpenCV.cc index 27509961..f01fd069 100755 --- a/src/OpenCV.cc +++ b/src/OpenCV.cc @@ -140,7 +140,7 @@ NAN_METHOD(OpenCV::ReadImageAsync) { argv[0] = Nan::Null(); argv[1] = Nan::Null(); - int callback_arg = -1; + int callback_arg = -1; int numargs = info.Length(); Local cb; @@ -257,13 +257,9 @@ NAN_METHOD(OpenCV::ReadImageAsync) { NAN_METHOD(OpenCV::ReadImage) { Nan::EscapableHandleScope scope; - Local argv[2]; argv[0] = Nan::Null(); - - Local im_h = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked(); - Matrix *img = Nan::ObjectWrap::Unwrap(im_h); - argv[1] = im_h; + argv[1] = Nan::Null(); int callback_arg = -1; int numargs = info.Length(); @@ -319,8 +315,7 @@ NAN_METHOD(OpenCV::ReadImage) { } - img->mat = mat; - Nan::AdjustExternalMemory(img->mat.dataend - img->mat.datastart); + argv[1] = Matrix::CreateWrappedFromMat(mat); } catch (cv::Exception& e) { argv[0] = Nan::Error(e.what()); argv[1] = Nan::Null(); @@ -335,7 +330,7 @@ NAN_METHOD(OpenCV::ReadImage) { } else { // if to return the mat if (success) - info.GetReturnValue().Set(im_h); + info.GetReturnValue().Set(argv[1]); else info.GetReturnValue().Set(Nan::New(false)); } From e1c0d8ef4e4aaca6eb217faac5df73a015d9a18d Mon Sep 17 00:00:00 2001 From: David Starke Date: Wed, 17 Jan 2018 14:55:57 -0800 Subject: [PATCH 7/7] Correctly track memory for image pyramid functions --- src/Matrix.cc | 6 ++++++ test/memory.js | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/Matrix.cc b/src/Matrix.cc index 1e6f2923..8cd71c35 100755 --- a/src/Matrix.cc +++ b/src/Matrix.cc @@ -2180,14 +2180,20 @@ NAN_METHOD(Matrix::WarpAffine) { NAN_METHOD(Matrix::PyrDown) { SETUP_FUNCTION(Matrix) + int oldSize = self->mat.dataend - self->mat.datastart; cv::pyrDown(self->mat, self->mat); + int newSize = self->mat.dataend - self->mat.datastart; + Nan::AdjustExternalMemory(newSize - oldSize); return; } NAN_METHOD(Matrix::PyrUp) { SETUP_FUNCTION(Matrix) + int oldSize = self->mat.dataend - self->mat.datastart; cv::pyrUp(self->mat, self->mat); + int newSize = self->mat.dataend - self->mat.datastart; + Nan::AdjustExternalMemory(newSize - oldSize); return; } diff --git a/test/memory.js b/test/memory.js index e414bb91..b1513451 100644 --- a/test/memory.js +++ b/test/memory.js @@ -825,6 +825,43 @@ test("Matrix reshape", t=>{ t.end(); }); +test("Matrix pyrDown", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + image.pyrDown(); + + t.equal(process.memoryUsage().external - startingMemory, 7500); //50 * 50 * 3 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + +test("Matrix pyrUp", t=>{ + gc(); + var startingMemory = process.memoryUsage().external; + + var image = new cv.Matrix(100, 100, cv.Constants.CV_8UC3, [0,0,0]); + t.equal(process.memoryUsage().external - startingMemory, 30000); //100 * 100 * 3 + + image.pyrUp(); + + t.equal(process.memoryUsage().external - startingMemory, 120000); //200 * 200 * 3 + + image.release(); + + var endingMemory = process.memoryUsage().external; + t.equal(endingMemory - startingMemory, 0); + t.end(); +}); + + //******************** // Additional Asynchronous Matrix Functions //********************