From 672b7cb49497a2780cb4764e1ed309793ec5efa4 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 3 Mar 2020 17:21:41 +0100 Subject: [PATCH 1/7] DataVisitor - a wrapper for obx_data_visitor --- lib/src/bindings/data_visitor.dart | 51 ++++++++++++++++++++++++++++++ lib/src/bindings/signatures.dart | 2 ++ 2 files changed, 53 insertions(+) create mode 100644 lib/src/bindings/data_visitor.dart diff --git a/lib/src/bindings/data_visitor.dart b/lib/src/bindings/data_visitor.dart new file mode 100644 index 000000000..3d0b6d937 --- /dev/null +++ b/lib/src/bindings/data_visitor.dart @@ -0,0 +1,51 @@ +import 'dart:ffi'; +import 'signatures.dart'; +import "package:ffi/ffi.dart" show allocate, free; + +typedef bool dataVisitorCallback(Pointer dataPtr, int length); + +int visitorId = 0; +final callbacks = {}; + +int _forwarder(Pointer callbackId, Pointer dataPtr, int size) { + if (callbackId == null) { + throw Exception("Data-visitor callback issued with NULL user_data"); + } + + return callbacks[callbackId.cast().value](dataPtr, size) ? 1 : 0; +} + +/// A data visitor wrapper/forwarder to be used where obx_data_visitor is expected. +class DataVisitor { + int _id; + Pointer _idPtr; + + Pointer> get fn => Pointer.fromFunction(_forwarder, 0); + + Pointer get userData => _idPtr.cast(); + + DataVisitor(dataVisitorCallback callback) { + // cycle through ids until we find an empty slot + visitorId++; + var initialId = visitorId; + while (callbacks.containsKey(visitorId)) { + visitorId++; + + if (initialId == visitorId) { + throw Exception("Data-visitor callbacks queue full - can't allocate another"); + } + } + // register the visitor + _id = visitorId; + callbacks[_id] = callback; + + _idPtr = allocate(); + _idPtr.value = _id; + } + + void close() { + // unregister the visitor + callbacks.remove(_id); + free(_idPtr); + } +} diff --git a/lib/src/bindings/signatures.dart b/lib/src/bindings/signatures.dart index a6ca625f6..281c67467 100644 --- a/lib/src/bindings/signatures.dart +++ b/lib/src/bindings/signatures.dart @@ -11,6 +11,8 @@ typedef obx_version_string_native_t = Pointer Function(); typedef obx_free_dart_t = void Function(Pointer ptr); typedef obx_free_native_t = Void Function(T ptr); // no Pointer, code analysis fails on usage +typedef obx_data_visitor_native_t = Uint8 Function(Pointer user_data, Pointer data, IntPtr size); + // error info typedef obx_last_error_code_native_t = Int32 Function(); typedef obx_last_error_message_native_t = Pointer Function(); From 989cfd422b354f9f844d29556bd731716fe458b8 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 3 Mar 2020 17:22:12 +0100 Subject: [PATCH 2/7] expose obx_supports_bytes_array internally --- lib/src/bindings/bindings.dart | 2 ++ lib/src/bindings/signatures.dart | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/src/bindings/bindings.dart b/lib/src/bindings/bindings.dart index 46f032b92..b7cfef034 100644 --- a/lib/src/bindings/bindings.dart +++ b/lib/src/bindings/bindings.dart @@ -13,6 +13,7 @@ class _ObjectBoxBindings { // common functions void Function(Pointer major, Pointer minor, Pointer patch) obx_version; Pointer Function() obx_version_string; + int Function() obx_supports_bytes_array; obx_free_dart_t obx_bytes_array_free; obx_free_dart_t obx_id_array_free; @@ -162,6 +163,7 @@ class _ObjectBoxBindings { // common functions obx_version = _fn("obx_version").asFunction(); obx_version_string = _fn("obx_version_string").asFunction(); + obx_supports_bytes_array = _fn("obx_supports_bytes_array").asFunction(); obx_bytes_array_free = _fn>>("obx_bytes_array_free").asFunction(); obx_id_array_free = _fn>>("obx_id_array_free").asFunction(); // obx_string_array_free = _fn>>("obx_string_array_free").asFunction(); diff --git a/lib/src/bindings/signatures.dart b/lib/src/bindings/signatures.dart index 281c67467..9247cc8b6 100644 --- a/lib/src/bindings/signatures.dart +++ b/lib/src/bindings/signatures.dart @@ -7,6 +7,7 @@ import 'package:ffi/ffi.dart'; // common functions typedef obx_version_native_t = Void Function(Pointer major, Pointer minor, Pointer patch); typedef obx_version_string_native_t = Pointer Function(); +typedef obx_supports_bytes_array_native_t = Uint8 Function(); typedef obx_free_dart_t = void Function(Pointer ptr); typedef obx_free_native_t = Void Function(T ptr); // no Pointer, code analysis fails on usage From 7559acf11db5b92d025b48c9b849a394b7e9bd77 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 3 Mar 2020 17:24:23 +0100 Subject: [PATCH 3/7] query.find() - large array support on 32-bit systems --- lib/src/bindings/signatures.dart | 10 +++++----- lib/src/query/query.dart | 25 ++++++++++++++++++++----- test/query_test.dart | 15 +++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/lib/src/bindings/signatures.dart b/lib/src/bindings/signatures.dart index 9247cc8b6..059634b1d 100644 --- a/lib/src/bindings/signatures.dart +++ b/lib/src/bindings/signatures.dart @@ -79,7 +79,7 @@ typedef obx_box_is_empty_native_t = Int32 Function(Pointer box, Pointer -> char[] // typedef Pointer -> int (e.g. obx_qb_cond); -// query builider +// query builder typedef obx_query_builder_native_t = Pointer Function(Pointer store, Uint32 entity_id); typedef obx_query_builder_dart_t = Pointer Function(Pointer store, int entity_id); @@ -150,10 +150,10 @@ typedef obx_query_count_dart_t = int Function(Pointer query, Pointer Function(Pointer query); -typedef obx_query_visit_native_t = Int32 Function( - Pointer query, Pointer visitor, Pointer user_data, Uint64 offset, Uint64 limit); -typedef obx_query_visit_dart_t = int Function( - Pointer query, Pointer visitor, Pointer user_data, int offset, int limit); +typedef obx_query_visit_native_t = Int32 Function(Pointer query, + Pointer> visitor, Pointer user_data, Uint64 offset, Uint64 limit); +typedef obx_query_visit_dart_t = int Function(Pointer query, + Pointer> visitor, Pointer user_data, int offset, int limit); // Utilities diff --git a/lib/src/query/query.dart b/lib/src/query/query.dart index 1518c8e55..abe48ae2d 100644 --- a/lib/src/query/query.dart +++ b/lib/src/query/query.dart @@ -7,6 +7,7 @@ import "../store.dart"; import "../common.dart"; import "../bindings/bindings.dart"; import "../bindings/constants.dart"; +import "../bindings/data_visitor.dart"; import "../bindings/flatbuffers.dart"; import "../bindings/helpers.dart"; import "../bindings/structs.dart"; @@ -571,11 +572,25 @@ class Query { List find({int offset = 0, int limit = 0}) { return _store.runInTransaction(TxMode.Read, () { - final bytesArray = checkObxPtr(bindings.obx_query_find(_cQuery, offset, limit), "find"); - try { - return _fbManager.unmarshalArray(bytesArray); - } finally { - bindings.obx_bytes_array_free(bytesArray); + if (bindings.obx_supports_bytes_array() == 1) { + final bytesArray = checkObxPtr(bindings.obx_query_find(_cQuery, offset, limit), "find"); + try { + return _fbManager.unmarshalArray(bytesArray); + } finally { + bindings.obx_bytes_array_free(bytesArray); + } + } else { + final results = []; + final visitor = DataVisitor((Pointer dataPtr, int length) { + final bytes = dataPtr.asTypedList(length); + results.add(_fbManager.unmarshal(bytes)); + return true; + }); + + final err = bindings.obx_query_visit(_cQuery, visitor.fn, visitor.userData, offset, limit); + visitor.close(); + checkObx(err); + return results; } }); } diff --git a/test/query_test.dart b/test/query_test.dart index 39d80478f..87446f487 100644 --- a/test/query_test.dart +++ b/test/query_test.dart @@ -204,6 +204,21 @@ void main() { q.close(); }); + test(".find works on large arrays", () { + // This would fail on 32-bit system if objectbox-c obx_supports_bytes_array() wasn't respected + final length = 10 * 1000; + final largeString = 'A' * length; + expect(largeString.length, length); + + box.put(TestEntity(tString: largeString)); + box.put(TestEntity(tString: largeString)); + + List items = box.query(TestEntity_.id.greaterThan(0)).build().find(); + expect(items.length, 2); + expect(items[0].tString, largeString); + expect(items[1].tString, largeString); + }); + test(".count items after grouping with and/or", () { box.put(TestEntity(tString: "Hello")); box.put(TestEntity(tString: "Goodbye")); From 2d948f7d2ba7eed87b431fcb4c57dccce9f6db01 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 4 Mar 2020 10:59:12 +0100 Subject: [PATCH 4/7] box getAll/getMany - large array support on 32-bit systems --- lib/src/bindings/bindings.dart | 6 ++++ lib/src/bindings/signatures.dart | 4 +++ lib/src/box.dart | 51 +++++++++++++++++++++++++------- test/box_test.dart | 24 ++++++++++++++- test/query_test.dart | 3 +- 5 files changed, 76 insertions(+), 12 deletions(-) diff --git a/lib/src/bindings/bindings.dart b/lib/src/bindings/bindings.dart index b7cfef034..10d7b72c1 100644 --- a/lib/src/bindings/bindings.dart +++ b/lib/src/bindings/bindings.dart @@ -69,6 +69,10 @@ class _ObjectBoxBindings { int Function(Pointer box, int id, Pointer> data, Pointer size) obx_box_get; Pointer Function(Pointer box, Pointer ids) obx_box_get_many; Pointer Function(Pointer box) obx_box_get_all; + int Function(Pointer box, Pointer ids, Pointer> visitor, + Pointer user_data) obx_box_visit_many; + int Function(Pointer box, Pointer> visitor, Pointer user_data) + obx_box_visit_all; int Function(Pointer box, int id_or_zero) obx_box_id_for_put; int Function(Pointer box, int count, Pointer out_first_id) obx_box_ids_for_put; int Function(Pointer box, int id, Pointer data, int size, int mode) obx_box_put; @@ -217,6 +221,8 @@ class _ObjectBoxBindings { obx_box_get = _fn("obx_box_get").asFunction(); obx_box_get_many = _fn("obx_box_get_many").asFunction(); obx_box_get_all = _fn("obx_box_get_all").asFunction(); + obx_box_visit_many = _fn("obx_box_visit_many").asFunction(); + obx_box_visit_all = _fn("obx_box_visit_all").asFunction(); obx_box_id_for_put = _fn("obx_box_id_for_put").asFunction(); obx_box_ids_for_put = _fn("obx_box_ids_for_put").asFunction(); obx_box_put = _fn("obx_box_put").asFunction(); diff --git a/lib/src/bindings/signatures.dart b/lib/src/bindings/signatures.dart index 059634b1d..6d668321d 100644 --- a/lib/src/bindings/signatures.dart +++ b/lib/src/bindings/signatures.dart @@ -61,6 +61,10 @@ typedef obx_box_get_native_t = Int32 Function( Pointer box, Uint64 id, Pointer> data, Pointer size); typedef obx_box_get_many_native_t = Pointer Function(Pointer box, Pointer ids); typedef obx_box_get_all_native_t = Pointer Function(Pointer box); +typedef obx_box_visit_many_native_t = Int32 Function(Pointer box, Pointer ids, + Pointer> visitor, Pointer user_data); +typedef obx_box_visit_all_native_t = Int32 Function( + Pointer box, Pointer> visitor, Pointer user_data); typedef obx_box_id_for_put_native_t = Uint64 Function(Pointer box, Uint64 id_or_zero); typedef obx_box_ids_for_put_native_t = Int32 Function(Pointer box, Uint64 count, Pointer out_first_id); typedef obx_box_put_native_t = Int32 Function( diff --git a/lib/src/box.dart b/lib/src/box.dart index 53594b0eb..bee974c92 100644 --- a/lib/src/box.dart +++ b/lib/src/box.dart @@ -5,6 +5,7 @@ import 'common.dart'; import "store.dart"; import "bindings/bindings.dart"; import "bindings/constants.dart"; +import "bindings/data_visitor.dart"; import "bindings/flatbuffers.dart"; import "bindings/helpers.dart"; import "bindings/structs.dart"; @@ -24,8 +25,9 @@ class Box { ModelEntity _modelEntity; ObjectReader _entityReader; OBXFlatbuffersManager _fbManager; + final bool _supportsBytesArrays; - Box(this._store) { + Box(this._store) : _supportsBytesArrays = bindings.obx_supports_bytes_array() == 1 { EntityDefinition entityDefs = _store.entityDef(); _modelEntity = entityDefs.model; _entityReader = entityDefs.reader; @@ -160,34 +162,63 @@ class Box { } } - List _getMany(bool allowMissing, Pointer Function() cCall) { + List _getMany( + bool allowMissing, Pointer Function() cGetArray, void Function(DataVisitor) cVisit) { return _store.runInTransaction(TxMode.Read, () { - final bytesArray = cCall(); + if (_supportsBytesArrays) { + final bytesArray = cGetArray(); + try { + return _fbManager.unmarshalArray(bytesArray, allowMissing: false); + } finally { + bindings.obx_bytes_array_free(bytesArray); + } + } + + final results = []; + final visitor = DataVisitor((Pointer dataPtr, int length) { + if (dataPtr == null || dataPtr.address == 0 || length == 0) { + if (allowMissing) { + results.add(null); + return true; + } else { + throw Exception('Object not found'); + } + } + final bytes = dataPtr.asTypedList(length); + results.add(_fbManager.unmarshal(bytes)); + return true; + }); + try { - return _fbManager.unmarshalArray(bytesArray, allowMissing: allowMissing); + cVisit(visitor); } finally { - bindings.obx_bytes_array_free(bytesArray); + visitor.close(); } + return results; }); } /// Returns a list of [ids.length] Objects of type T, each corresponding to the location of its ID in [ids]. - /// Non-existant IDs become null. + /// Non-existent IDs become null. List getMany(List ids) { if (ids.isEmpty) return []; - const bool allowMissing = true; // returns null if null is encountered in the data found + const bool allowMissing = true; // result includes null if an object is missing return OBX_id_array.executeWith( ids, - (ptr) => _getMany(allowMissing, - () => checkObxPtr(bindings.obx_box_get_many(_cBox, ptr), "failed to get many objects from box"))); + (ptr) => _getMany( + allowMissing, + () => checkObxPtr(bindings.obx_box_get_many(_cBox, ptr), "failed to get many objects from box"), + (DataVisitor visitor) => checkObx(bindings.obx_box_visit_many(_cBox, ptr, visitor.fn, visitor.userData)))); } /// Returns all stored objects in this Box. List getAll() { const bool allowMissing = false; // throw if null is encountered in the data found return _getMany( - allowMissing, () => checkObxPtr(bindings.obx_box_get_all(_cBox), "failed to get all objects from box")); + allowMissing, + () => checkObxPtr(bindings.obx_box_get_all(_cBox), "failed to get all objects from box"), + (DataVisitor visitor) => checkObx(bindings.obx_box_visit_all(_cBox, visitor.fn, visitor.userData))); } /// Returns a builder to create queries for Object matching supplied criteria. diff --git a/test/box_test.dart b/test/box_test.dart index 92f36ef31..374e5257c 100644 --- a/test/box_test.dart +++ b/test/box_test.dart @@ -78,7 +78,29 @@ void main() { } }); - test(".getMany correctly handles non-existant items", () { + test(".getAll/getMany works on large arrays", () { + // This would fail on 32-bit system if objectbox-c obx_supports_bytes_array() wasn't respected + final length = 10 * 1000; + final largeString = 'A' * length; + expect(largeString.length, length); + + box.put(TestEntity(tString: largeString)); + box.put(TestEntity(tString: largeString)); + + List items = box.getAll(); + expect(items.length, 2); + expect(items[0].tString, largeString); + expect(items[1].tString, largeString); + + box.put(TestEntity(tString: largeString)); + + items = box.getMany([1, 2]); + expect(items.length, 2); + expect(items[0].tString, largeString); + expect(items[1].tString, largeString); + }); + + test(".getMany correctly handles non-existent items", () { final List items = ["One", "Two"].map((s) => TestEntity(tString: s)).toList(); final List ids = box.putMany(items); int otherId = 1; diff --git a/test/query_test.dart b/test/query_test.dart index 87446f487..79e309d15 100644 --- a/test/query_test.dart +++ b/test/query_test.dart @@ -210,10 +210,11 @@ void main() { final largeString = 'A' * length; expect(largeString.length, length); + box.put(TestEntity(tString: largeString)); box.put(TestEntity(tString: largeString)); box.put(TestEntity(tString: largeString)); - List items = box.query(TestEntity_.id.greaterThan(0)).build().find(); + List items = box.query(TestEntity_.id.lessThan(3)).build().find(); expect(items.length, 2); expect(items[0].tString, largeString); expect(items[1].tString, largeString); From f44023aad081f77fec1aed4bd75328028f194f5d Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 4 Mar 2020 11:30:42 +0100 Subject: [PATCH 5/7] hide data visitor internals --- lib/src/bindings/data_visitor.dart | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/src/bindings/data_visitor.dart b/lib/src/bindings/data_visitor.dart index 3d0b6d937..d79b15317 100644 --- a/lib/src/bindings/data_visitor.dart +++ b/lib/src/bindings/data_visitor.dart @@ -2,17 +2,16 @@ import 'dart:ffi'; import 'signatures.dart'; import "package:ffi/ffi.dart" show allocate, free; -typedef bool dataVisitorCallback(Pointer dataPtr, int length); - -int visitorId = 0; -final callbacks = {}; +int _lastId = 0; +final _callbacks = dataPtr, int length)>{}; +// called from C, forwards calls to the actual callback registered at the given ID int _forwarder(Pointer callbackId, Pointer dataPtr, int size) { - if (callbackId == null) { - throw Exception("Data-visitor callback issued with NULL user_data"); + if (callbackId == null || callbackId.address == 0) { + throw Exception("Data-visitor callback issued with NULL user_data (callback ID)"); } - return callbacks[callbackId.cast().value](dataPtr, size) ? 1 : 0; + return _callbacks[callbackId.cast().value](dataPtr, size) ? 1 : 0; } /// A data visitor wrapper/forwarder to be used where obx_data_visitor is expected. @@ -24,20 +23,20 @@ class DataVisitor { Pointer get userData => _idPtr.cast(); - DataVisitor(dataVisitorCallback callback) { + DataVisitor(bool Function(Pointer dataPtr, int length) callback) { // cycle through ids until we find an empty slot - visitorId++; - var initialId = visitorId; - while (callbacks.containsKey(visitorId)) { - visitorId++; + _lastId++; + var initialId = _lastId; + while (_callbacks.containsKey(_lastId)) { + _lastId++; - if (initialId == visitorId) { + if (initialId == _lastId) { throw Exception("Data-visitor callbacks queue full - can't allocate another"); } } // register the visitor - _id = visitorId; - callbacks[_id] = callback; + _id = _lastId; + _callbacks[_id] = callback; _idPtr = allocate(); _idPtr.value = _id; @@ -45,7 +44,7 @@ class DataVisitor { void close() { // unregister the visitor - callbacks.remove(_id); + _callbacks.remove(_id); free(_idPtr); } } From 3cf705d22f68307f63157672de46ffca8c1740fa Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 4 Mar 2020 11:35:15 +0100 Subject: [PATCH 6/7] restore box getMany allowMissing behaviour when not using data-visitors --- lib/src/box.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/box.dart b/lib/src/box.dart index bee974c92..0919f4866 100644 --- a/lib/src/box.dart +++ b/lib/src/box.dart @@ -168,7 +168,7 @@ class Box { if (_supportsBytesArrays) { final bytesArray = cGetArray(); try { - return _fbManager.unmarshalArray(bytesArray, allowMissing: false); + return _fbManager.unmarshalArray(bytesArray, allowMissing: allowMissing); } finally { bindings.obx_bytes_array_free(bytesArray); } From 713d59a5fd991dcddf5dc9d2e188ed326b85c07f Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 5 Mar 2020 17:49:29 +0100 Subject: [PATCH 7/7] DataVisitor docs and minor formatting changes --- lib/src/bindings/data_visitor.dart | 22 ++++++++++++++++ lib/src/box.dart | 40 +++++++++++++++--------------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/lib/src/bindings/data_visitor.dart b/lib/src/bindings/data_visitor.dart index d79b15317..eb8b379ff 100644 --- a/lib/src/bindings/data_visitor.dart +++ b/lib/src/bindings/data_visitor.dart @@ -2,6 +2,28 @@ import 'dart:ffi'; import 'signatures.dart'; import "package:ffi/ffi.dart" show allocate, free; +/// This file implements C call forwarding using a trampoline approach. +/// +/// When you want to pass a dart callback to a C function you cannot use lambdas and instead the callback must be +/// a static function, otherwise `Pointer.fromFunction()` called with your function won't compile. +/// Since static functions don't have any state, you must either rely on a global state or use a "userData" pointer +/// pass-through functionality provided by a C function. +/// +/// The DataVisitor class tries to alleviate the burden of managing this and instead allows using lambdas from +/// user-code, internally mapping the C calls to the appropriate lambda. +/// +/// Sample usage: +/// final results = []; +/// final visitor = DataVisitor((Pointer dataPtr, int length) { +/// final bytes = dataPtr.asTypedList(length); +/// results.add(_fbManager.unmarshal(bytes)); +/// return true; // return value usually indicates to the C function whether it should continue. +/// }); +/// +/// final err = bindings.obx_query_visit(_cQuery, visitor.fn, visitor.userData, offset, limit); +/// visitor.close(); // make sure to close the visitor, unregistering the callback it from the forwarder +/// checkObx(err); + int _lastId = 0; final _callbacks = dataPtr, int length)>{}; diff --git a/lib/src/box.dart b/lib/src/box.dart index 0919f4866..d0d6316be 100644 --- a/lib/src/box.dart +++ b/lib/src/box.dart @@ -172,29 +172,29 @@ class Box { } finally { bindings.obx_bytes_array_free(bytesArray); } - } - - final results = []; - final visitor = DataVisitor((Pointer dataPtr, int length) { - if (dataPtr == null || dataPtr.address == 0 || length == 0) { - if (allowMissing) { - results.add(null); - return true; - } else { - throw Exception('Object not found'); + } else { + final results = []; + final visitor = DataVisitor((Pointer dataPtr, int length) { + if (dataPtr == null || dataPtr.address == 0 || length == 0) { + if (allowMissing) { + results.add(null); + return true; + } else { + throw Exception('Object not found'); + } } - } - final bytes = dataPtr.asTypedList(length); - results.add(_fbManager.unmarshal(bytes)); - return true; - }); + final bytes = dataPtr.asTypedList(length); + results.add(_fbManager.unmarshal(bytes)); + return true; + }); - try { - cVisit(visitor); - } finally { - visitor.close(); + try { + cVisit(visitor); + } finally { + visitor.close(); + } + return results; } - return results; }); }