Skip to content

Track external memory usage for matrices #602

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 4, 2018
Merged

Track external memory usage for matrices #602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
dbf2df0
Track external memory usage for matrices
dstarke Dec 22, 2017
dff9971
missed adjusting the external memory on resize
dstarke Jan 4, 2018
f8d7dc2
More accurate tracking of external memory using OpenCV’s reference co…
dstarke Jan 5, 2018
2cd3bd4
slightly more clear memory handling for async tasks
dstarke Jan 5, 2018
45c8894
Additional memory tracking for matrices
dstarke Jan 11, 2018
9646626
Consolidate external memory adjustment in Matrix
dstarke Jan 17, 2018
e1c0d8e
Correctly track memory for image pyramid functions
dstarke Jan 17, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 14 additions & 21 deletions src/BackgroundSubtractor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,10 +255,6 @@ NAN_METHOD(BackgroundSubtractorWrap::ApplyMOG) {


try {
Local<Object> fgMask =
Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *img = Nan::ObjectWrap::Unwrap<Matrix>(fgMask);

cv::Mat mat;
if (Buffer::HasInstance(info[0])) {
uint8_t *buf = (uint8_t *) Buffer::Data(info[0]->ToObject());
Expand Down Expand Up @@ -287,7 +283,7 @@ NAN_METHOD(BackgroundSubtractorWrap::ApplyMOG) {
#endif
}

img->mat = _fgMask;
Local<Object> fgMask = Matrix::CreateWrappedFromMat(_fgMask.clone());
mat.release();

argv[0] = Nan::Null();
Expand All @@ -313,15 +309,15 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {
AsyncBackgroundSubtractorWorker(
Nan::Callback *callback,
BackgroundSubtractorWrap *bg,
cv::Mat &img_mat):
Matrix *matrix_in):
Nan::AsyncWorker(callback),
bg(bg),
img_mat(img_mat) { // note: this makes a new cv::Mat, and so increments the ref count for the data without copying it
matrix(new Matrix(matrix_in)) {

}

~AsyncBackgroundSubtractorWorker() {
// upon destroy, img_mat will reduce refcount on data by one

}

// Executed inside the worker-thread.
Expand All @@ -332,9 +328,9 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {
// wait here if already in apply - auto-release on scope exit
BGAutoMutex(bg->applymutex);
#if CV_MAJOR_VERSION >= 3
bg->subtractor->apply(this->img_mat, _fgMask);
bg->subtractor->apply(matrix->mat, _fgMask);
#else
bg->subtractor->operator()(this->img_mat, _fgMask);
bg->subtractor->operator()(matrix->mat, _fgMask);
#endif
}

Expand All @@ -344,9 +340,10 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {
void HandleOKCallback() {
Nan::HandleScope scope;

Local<Object> im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *imgout = Nan::ObjectWrap::Unwrap<Matrix>(im_to_return);
imgout->mat = _fgMask;
delete matrix;
matrix = NULL;

Local<Object> im_to_return = Matrix::CreateWrappedFromMat(_fgMask.clone());

Local<Value> argv[] = {
Nan::Null()
Expand All @@ -362,7 +359,7 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {

private:
BackgroundSubtractorWrap *bg;
cv::Mat img_mat;
Matrix *matrix;
cv::Mat _fgMask;
};

Expand All @@ -372,7 +369,6 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) {
SETUP_FUNCTION(BackgroundSubtractorWrap);
int callback_arg = -1;
int numargs = info.Length();
int success = 1;

Local<Function> cb;

Expand Down Expand Up @@ -402,15 +398,12 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) {

Nan::Callback *callback = new Nan::Callback(cb.As<Function>());
Matrix *_img = Nan::ObjectWrap::Unwrap<Matrix>(info[0]->ToObject());
Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, _img->mat));
Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, _img));
return;
} else { //synchronous - return the image

try {
Local<Object> fgMask =
Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *img = Nan::ObjectWrap::Unwrap<Matrix>(fgMask);

Local<Object> fgMask;
cv::Mat mat;
if (Buffer::HasInstance(info[0])) {
uint8_t *buf = (uint8_t *) Buffer::Data(info[0]->ToObject());
Expand All @@ -436,7 +429,7 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) {
#else
self->subtractor->operator()(mat, _fgMask);
#endif
img->mat = _fgMask;
fgMask = Matrix::CreateWrappedFromMat(_fgMask.clone());
}

mat.release();
Expand Down
47 changes: 19 additions & 28 deletions src/Calib3D.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,6 @@

#ifdef HAVE_OPENCV_CALIB3D

inline Local<Object> matrixFromMat(cv::Mat &input) {
Local<Object> matrixWrap =
Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *matrix = Nan::ObjectWrap::Unwrap<Matrix>(matrixWrap);
matrix->mat = input;

return matrixWrap;
}

inline cv::Mat matFromMatrix(Local<Value> matrix) {
Matrix* m = Nan::ObjectWrap::Unwrap<Matrix>(matrix->ToObject());
return m->mat;
Expand Down Expand Up @@ -237,11 +228,11 @@ NAN_METHOD(Calib3D::CalibrateCamera) {
ret->Set(Nan::New<String>("reprojectionError").ToLocalChecked(), Nan::New<Number>(error));

// K
Local<Object> KMatrixWrap = matrixFromMat(K);
Local<Object> KMatrixWrap = Matrix::CreateWrappedFromMat(K);
ret->Set(Nan::New<String>("K").ToLocalChecked(), KMatrixWrap);

// dist
Local<Object> distMatrixWrap = matrixFromMat(dist);
Local<Object> distMatrixWrap = Matrix::CreateWrappedFromMat(dist);
ret->Set(Nan::New<String>("distortion").ToLocalChecked(), distMatrixWrap);

// Per frame R and t, skiping for now
Expand Down Expand Up @@ -287,11 +278,11 @@ NAN_METHOD(Calib3D::SolvePnP) {
Local<Object> ret = Nan::New<Object>();

// rvec
Local<Object> rMatrixWrap = matrixFromMat(rvec);
Local<Object> rMatrixWrap = Matrix::CreateWrappedFromMat(rvec);
ret->Set(Nan::New<String>("rvec").ToLocalChecked(), rMatrixWrap);

// tvec
Local<Object> tMatrixWrap = matrixFromMat(tvec);
Local<Object> tMatrixWrap = Matrix::CreateWrappedFromMat(tvec);
ret->Set(Nan::New<String>("tvec").ToLocalChecked(), tMatrixWrap);

// Return
Expand Down Expand Up @@ -334,7 +325,7 @@ NAN_METHOD(Calib3D::GetOptimalNewCameraMatrix) {
newImageSize);

// Wrap the output K
Local<Object> KMatrixWrap = matrixFromMat(Kout);
Local<Object> KMatrixWrap = Matrix::CreateWrappedFromMat(Kout);

// Return the new K matrix
info.GetReturnValue().Set(KMatrixWrap);
Expand Down Expand Up @@ -394,28 +385,28 @@ NAN_METHOD(Calib3D::StereoCalibrate) {
// Make the output arguments

// k1
Local<Object> K1MatrixWrap = matrixFromMat(k1);
Local<Object> K1MatrixWrap = Matrix::CreateWrappedFromMat(k1);

// d1
Local<Object> d1MatrixWrap = matrixFromMat(d1);
Local<Object> d1MatrixWrap = Matrix::CreateWrappedFromMat(d1);

// k2
Local<Object> K2MatrixWrap = matrixFromMat(k2);
Local<Object> K2MatrixWrap = Matrix::CreateWrappedFromMat(k2);

// d2
Local<Object> d2MatrixWrap = matrixFromMat(d2);
Local<Object> d2MatrixWrap = Matrix::CreateWrappedFromMat(d2);

// R
Local<Object> RMatrixWrap = matrixFromMat(R);
Local<Object> RMatrixWrap = Matrix::CreateWrappedFromMat(R);

// t
Local<Object> tMatrixWrap = matrixFromMat(t);
Local<Object> tMatrixWrap = Matrix::CreateWrappedFromMat(t);

// E
Local<Object> EMatrixWrap = matrixFromMat(E);
Local<Object> EMatrixWrap = Matrix::CreateWrappedFromMat(E);

// F
Local<Object> FMatrixWrap = matrixFromMat(F);
Local<Object> FMatrixWrap = Matrix::CreateWrappedFromMat(F);

// Add to return object
ret->Set(Nan::New<String>("K1").ToLocalChecked(), K1MatrixWrap);
Expand Down Expand Up @@ -479,11 +470,11 @@ NAN_METHOD(Calib3D::StereoRectify) {
// Make the return object
Local<Object> ret = Nan::New<Object>();

ret->Set(Nan::New<String>("R1").ToLocalChecked(), matrixFromMat(R1));
ret->Set(Nan::New<String>("R2").ToLocalChecked(), matrixFromMat(R2));
ret->Set(Nan::New<String>("P1").ToLocalChecked(), matrixFromMat(P1));
ret->Set(Nan::New<String>("P2").ToLocalChecked(), matrixFromMat(P2));
ret->Set(Nan::New<String>("Q").ToLocalChecked(), matrixFromMat(Q));
ret->Set(Nan::New<String>("R1").ToLocalChecked(), Matrix::CreateWrappedFromMat(R1));
ret->Set(Nan::New<String>("R2").ToLocalChecked(), Matrix::CreateWrappedFromMat(R2));
ret->Set(Nan::New<String>("P1").ToLocalChecked(), Matrix::CreateWrappedFromMat(P1));
ret->Set(Nan::New<String>("P2").ToLocalChecked(), Matrix::CreateWrappedFromMat(P2));
ret->Set(Nan::New<String>("Q").ToLocalChecked(), Matrix::CreateWrappedFromMat(Q));

// Return the rectification parameters
info.GetReturnValue().Set(ret);
Expand Down Expand Up @@ -557,7 +548,7 @@ NAN_METHOD(Calib3D::ReprojectImageTo3D) {
cv::reprojectImageTo3D(disparity, depthImage, Q);

// Wrap the depth image
Local<Object> depthImageMatrix = matrixFromMat(depthImage);
Local<Object> depthImageMatrix = Matrix::CreateWrappedFromMat(depthImage);

info.GetReturnValue().Set(depthImageMatrix);
} catch (cv::Exception &e) {
Expand Down
6 changes: 4 additions & 2 deletions src/CascadeClassifierWrap.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ class AsyncDetectMultiScale: public Nan::AsyncWorker {
Matrix* im, double scale, int neighbors, int minw, int minh) :
Nan::AsyncWorker(callback),
cc(cc),
im(im),
im(new Matrix(im)), //copy the matrix so we aren't affected if the original is released
scale(scale),
neighbors(neighbors),
minw(minw),
Expand Down Expand Up @@ -82,7 +82,9 @@ class AsyncDetectMultiScale: public Nan::AsyncWorker {

void HandleOKCallback() {
Nan::HandleScope scope;
// this->matrix->Unref();

delete im;
im = NULL;

Local < Value > argv[2];
v8::Local < v8::Array > arr = Nan::New < v8::Array > (this->res.size());
Expand Down
49 changes: 36 additions & 13 deletions src/FaceRecognizer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ namespace cv {
#define FISHER 2

// Todo, move somewhere useful
// Note: References to the returned object here should not be retained past the end of the calling function.
// Otherwise, node might not keep track of external memory usage correctly.
cv::Mat fromMatrixOrFilename(Local<Value> v) {
cv::Mat im;
if (v->IsString()) {
Expand All @@ -40,6 +42,20 @@ cv::Mat fromMatrixOrFilename(Local<Value> v) {
return im;
}

// Note: Use this function when you might need to retain the returned object past the end of the calling function,
// such as in asynchronous methods
Matrix *CreateFromMatrixOrFilename(Local<Value> v) {
if (v->IsString()) {
Matrix *im = new Matrix();
std::string filename = std::string(*Nan::Utf8String(v->ToString()));
im->setMat(cv::imread(filename));
return im;
// std::cout<< im.size();
} else {
return new Matrix(Nan::ObjectWrap::Unwrap<Matrix>(v->ToObject()));
}
}

Nan::Persistent<FunctionTemplate> FaceRecognizerWrap::constructor;

void FaceRecognizerWrap::Init(Local<Object> target) {
Expand Down Expand Up @@ -188,7 +204,7 @@ Local<Value> UnwrapTrainingData(Nan::NAN_METHOD_ARGS_TYPE info,
}

int label = valarr->Get(0)->Uint32Value();
cv::Mat im = fromMatrixOrFilename(valarr->Get(1));
cv::Mat im = fromMatrixOrFilename(valarr->Get(1)); //this is ok because we clone the image
im = im.clone();
if (im.channels() == 3) {
cv::cvtColor(im, im, CV_RGB2GRAY);
Expand Down Expand Up @@ -295,7 +311,9 @@ NAN_METHOD(FaceRecognizerWrap::PredictSync) {

cv::Mat im = fromMatrixOrFilename(info[0]); // TODO CHECK!
if (im.channels() == 3) {
cv::cvtColor(im, im, CV_RGB2GRAY);
cv::Mat previous = im;
im = cv::Mat();
cv::cvtColor(previous, im, CV_RGB2GRAY);
}

int predictedLabel = -1;
Expand All @@ -322,10 +340,10 @@ NAN_METHOD(FaceRecognizerWrap::PredictSync) {

class PredictASyncWorker: public Nan::AsyncWorker {
public:
PredictASyncWorker(Nan::Callback *callback, cv::Ptr<cv::FaceRecognizer> rec, cv::Mat im) :
PredictASyncWorker(Nan::Callback *callback, cv::Ptr<cv::FaceRecognizer> rec, Matrix *matrix_in) :
Nan::AsyncWorker(callback),
rec(rec),
im(im) {
matrix(new Matrix(matrix_in)) {
predictedLabel = -1;
confidence = 0.0;
}
Expand All @@ -334,7 +352,7 @@ class PredictASyncWorker: public Nan::AsyncWorker {
}

void Execute() {
this->rec->predict(this->im, this->predictedLabel, this->confidence);
rec->predict(matrix->mat, predictedLabel, confidence);
#if CV_MAJOR_VERSION >= 3
// Older versions of OpenCV3 incorrectly returned label=0 at
// confidence=DBL_MAX instead of label=-1 on failure. This can be removed
Expand All @@ -350,6 +368,9 @@ class PredictASyncWorker: public Nan::AsyncWorker {
void HandleOKCallback() {
Nan::HandleScope scope;

delete matrix;
matrix = NULL;

v8::Local<v8::Object> res = Nan::New<Object>();
res->Set(Nan::New("id").ToLocalChecked(), Nan::New<Number>(predictedLabel));
res->Set(Nan::New("confidence").ToLocalChecked(), Nan::New<Number>(confidence));
Expand All @@ -367,7 +388,7 @@ class PredictASyncWorker: public Nan::AsyncWorker {

private:
cv::Ptr<cv::FaceRecognizer> rec;
cv::Mat im;
Matrix *matrix;
int predictedLabel;
double confidence;
};
Expand All @@ -381,13 +402,17 @@ NAN_METHOD(FaceRecognizerWrap::Predict) {

REQ_FUN_ARG(1, cb);

cv::Mat im = fromMatrixOrFilename(info[0]);
if (im.channels() == 3) {
cv::cvtColor(im, im, CV_RGB2GRAY);
Matrix *m = CreateFromMatrixOrFilename(info[0]);
if (m->mat.channels() == 3) {
cv::Mat grayMat;
cv::cvtColor(m->mat, grayMat, CV_RGB2GRAY);
m->setMat(grayMat);
}

Nan::Callback *callback = new Nan::Callback(cb.As<Function>());
Nan::AsyncQueueWorker(new PredictASyncWorker(callback, self->rec, im));
Nan::AsyncQueueWorker(new PredictASyncWorker(callback, self->rec, m));

delete m;

return;
}
Expand Down Expand Up @@ -448,9 +473,7 @@ NAN_METHOD(FaceRecognizerWrap::GetMat) {
m = self->rec->getMat(key);
#endif

Local<Object> im = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *img = Nan::ObjectWrap::Unwrap<Matrix>(im);
img->mat = m;
Local<Object> im = Matrix::CreateWrappedFromMat(m);

info.GetReturnValue().Set(im);
}
Expand Down
Loading