From b5ed9760aa5b030fa8692b676a8e238cbffb81a7 Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Wed, 23 Jul 2025 15:20:53 -0400 Subject: [PATCH 1/9] Changed remove to now remove the first occurance of an element, in the chance there are duplicate entries. --- include/GenSync/Data/DataContainer.h | 4 ++-- include/GenSync/Data/InMemContainer.h | 2 +- src/Data/InMemContainer.cpp | 7 +++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/GenSync/Data/DataContainer.h b/include/GenSync/Data/DataContainer.h index bb8dd28..3ec8ebf 100644 --- a/include/GenSync/Data/DataContainer.h +++ b/include/GenSync/Data/DataContainer.h @@ -190,12 +190,12 @@ class DataContainer{ virtual void clear() = 0; /** - * Removes all DataObjects that contain the internal data as the given DataObject. + * Removes the first DataObject that contain the internal data as the given DataObject. * The internal data refers to the information that the DataObject represents. * @param val The given DataObject. * @return Returns true if object is successfully removed. */ - virtual bool remove (const shared_ptr& val) = 0; + virtual bool remove(const shared_ptr& val) = 0; /** * Pushes a given DataObject into the container for storage. diff --git a/include/GenSync/Data/InMemContainer.h b/include/GenSync/Data/InMemContainer.h index 90936cd..1dfa605 100644 --- a/include/GenSync/Data/InMemContainer.h +++ b/include/GenSync/Data/InMemContainer.h @@ -127,7 +127,7 @@ class InMemContainer : public DataContainer{ void clear() override; /** - * Removes all DataObjects that contain the internal data as the given DataObject. + * Removes the first DataObject that contain the internal data as the given DataObject. * The internal data refers to the information that the DataObject represents. * @param val The given DataObject. * @return Returns true if object is successfully removed. diff --git a/src/Data/InMemContainer.cpp b/src/Data/InMemContainer.cpp index 656e662..8ccd47b 100644 --- a/src/Data/InMemContainer.cpp +++ b/src/Data/InMemContainer.cpp @@ -31,8 +31,11 @@ void InMemContainer::clear(){ } bool InMemContainer::remove (const shared_ptr& val){ - int before =myData.size(); - myData.remove(val); + int before = myData.size(); + auto it = find(myData.begin(), myData.end(), val); + if(it != myData.end()){ + myData.erase(it); + } return myData.size() < before; } From 7e46011ecfd843368c37da132f5cb1435b99a7d4 Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Wed, 23 Jul 2025 15:24:39 -0400 Subject: [PATCH 2/9] Renaming add test for InMemContainerTest --- tests/unit/InMemContainerTest.cpp | 2 +- tests/unit/InMemContainerTest.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/InMemContainerTest.cpp b/tests/unit/InMemContainerTest.cpp index 16a83ab..cc8f062 100644 --- a/tests/unit/InMemContainerTest.cpp +++ b/tests/unit/InMemContainerTest.cpp @@ -9,7 +9,7 @@ void InMemContainerTest::setUp(){ srand(SEED); } -void InMemContainerTest::addIterateTest(){ +void InMemContainerTest::addTest(){ //Create containers list> objList; InMemContainer container; diff --git a/tests/unit/InMemContainerTest.h b/tests/unit/InMemContainerTest.h index bee0dd1..09fc438 100644 --- a/tests/unit/InMemContainerTest.h +++ b/tests/unit/InMemContainerTest.h @@ -7,7 +7,7 @@ class InMemContainerTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(InMemContainerTest); - CPPUNIT_TEST(addIterateTest); + CPPUNIT_TEST(addTest); CPPUNIT_TEST(clearTest); CPPUNIT_TEST(emptyTest); CPPUNIT_TEST(sizeTest); @@ -51,7 +51,7 @@ class InMemContainerTest : public CppUnit::TestFixture { * Adds elements to a container and iterates through them using a list * as comparison to ensure values have been successfully inserted. */ - void addIterateTest(); + void addTest(); /** * Tests the clear function by repeatedly filling and clearing the container. From c364f44fc63936aa50d81afa7fa4957fcd1e4184 Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Mon, 28 Jul 2025 11:48:40 -0400 Subject: [PATCH 3/9] SQLiteContainer Implementation --- CMakeLists.txt | 44 +++- include/GenSync/Aux/UID.h | 5 + include/GenSync/Data/DataContainer.h | 5 +- include/GenSync/Data/DatabaseContainer.h | 74 +++++++ include/GenSync/Data/InMemContainer.h | 3 + include/GenSync/Data/SQLiteContainer.h | 180 +++++++++++++++++ include/GenSync/Syncs/GenSync.h | 70 +++++++ src/Data/InMemContainer.cpp | 3 + src/Data/SQLiteContainer.cpp | 243 +++++++++++++++++++++++ src/Syncs/GenSync.cpp | 37 +++- src/TryMe.cpp | 53 ++--- testDB | 0 tests/TestAuxiliary.h | 135 +++++++++++++ tests/unit/InMemContainerTest.cpp | 90 +-------- tests/unit/InMemContainerTest.h | 20 -- tests/unit/SQLiteContainerTest.cpp | 47 +++++ tests/unit/SQLiteContainerTest.h | 59 ++++++ 17 files changed, 927 insertions(+), 141 deletions(-) create mode 100644 include/GenSync/Data/DatabaseContainer.h create mode 100644 include/GenSync/Data/SQLiteContainer.h create mode 100644 src/Data/SQLiteContainer.cpp create mode 100644 testDB create mode 100644 tests/unit/SQLiteContainerTest.cpp create mode 100644 tests/unit/SQLiteContainerTest.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e52890..4bd2472 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,8 @@ endif() include(GNUInstallDirs) +option(USE_SQLITE "Enable SQLite support" OFF) + # Set project directory strucuture set(SRC_DIR src) set(INCLUDE_DIR include) @@ -102,6 +104,7 @@ set(HEADERS ${DATA_DIR_INC}/DataPriorityObject.h ${DATA_DIR_INC}/DataContainer.h ${DATA_DIR_INC}/InMemContainer.h + ${DATA_DIR_INC}/DatabaseContainer.h ${COMM_DIR_INC}/CommSocket.h ${COMM_DIR_INC}/CommString.h @@ -141,11 +144,25 @@ set(HEADERS ${SYNC_BENCH_INC}/FromFileGen.h ) -# Add gensync library +if(USE_SQLITE STREQUAL "ON") + message(STATUS "SQLite support ENABLED") + list(APPEND SOURCE_FILES ${DATA_DIR}/SQLiteContainer.cpp) + list(APPEND HEADERS ${DATA_DIR_INC}/SQLiteContainer.h) + set(USE_SQLITE_DEFINE USE_SQLITE) +else() + message(STATUS "SQLite support DISABLED") + set(USE_SQLITE_DEFINE "") +endif() + +# Now create gensync library AFTER conditionals add_library(gensync STATIC ${SOURCE_FILES} ${HEADERS}) target_include_directories(gensync PUBLIC ${INCLUDE_DIR}) target_compile_definitions(gensync PUBLIC RECORD="${RECORD_DIR}") target_compile_definitions(gensync PUBLIC DEFAULT_LOGLEVEL=${DEFAULT_LOG_LEVEL}) +if(USE_SQLITE) + target_compile_definitions(gensync PUBLIC ${USE_SQLITE_DEFINE}) + target_link_libraries(gensync PUBLIC sqlite3) +endif() # include Apache Data Sketches as a header-only library target_link_libraries(gensync PUBLIC ntl pthread gmp) @@ -182,19 +199,34 @@ target_link_libraries(EncodeJoin gensync) macro(add_group_test dir name) file(GLOB testPaths ${dir}/*Test.cpp ${dir}/*Tests.cpp) - # add one executable with all tests - add_executable(${name} ${TEST_RUNNER} ${testPaths}) + set(filteredTestPaths "") + + foreach(test ${testPaths}) + get_filename_component(testFileName ${test} NAME) + + # Skip SQLiteContainerTest if USE_SQLITE is OFF + if(testFileName MATCHES "SQLiteContainerTest.cpp") + if(USE_SQLITE) + list(APPEND filteredTestPaths ${test}) + endif() + else() + list(APPEND filteredTestPaths ${test}) + endif() + endforeach() + + # Add one executable with all tests + add_executable(${name} ${TEST_RUNNER} ${filteredTestPaths}) target_link_libraries(${name} gensync cppunit) target_include_directories(${name} PRIVATE tests) - # add an executable for each test file and register it - foreach(test ${testPaths}) + # Add an executable for each test file and register it + foreach(test ${filteredTestPaths}) get_filename_component(testName ${test} NAME_WE) add_executable(${testName} ${TEST_RUNNER} ${test}) target_link_libraries(${testName} gensync cppunit) target_include_directories(${testName} PRIVATE tests) add_test(${testName} ${testName}) - endforeach(test) + endforeach() endmacro() if(BUILD_TESTS) diff --git a/include/GenSync/Aux/UID.h b/include/GenSync/Aux/UID.h index b2c0840..e75c5c8 100644 --- a/include/GenSync/Aux/UID.h +++ b/include/GenSync/Aux/UID.h @@ -31,6 +31,11 @@ class UID { return myID; } + void setObjectID(int newID){ + myID = newID; + } + + private: int myID; /** the ID of this object */ static int ID_count; /** Maintains a count of the number of UIDs created in the program thus far. */ diff --git a/include/GenSync/Data/DataContainer.h b/include/GenSync/Data/DataContainer.h index 3ec8ebf..16ecdbe 100644 --- a/include/GenSync/Data/DataContainer.h +++ b/include/GenSync/Data/DataContainer.h @@ -1,4 +1,7 @@ /* This code is part of the GenSync project developed at Boston University. Please see the README for use and references. */ +// Created by GregoryFan on 7/10/2025 +// + #ifndef DATACONTAINER_H #define DATACONTAINER_H #include @@ -170,7 +173,7 @@ class DataContainer{ virtual const_iterator begin() const = 0; /** - * @return A const iterator that points to the beginning of the container. + * @return A const iterator that points to the final item of the container. */ virtual const_iterator end() const = 0; diff --git a/include/GenSync/Data/DatabaseContainer.h b/include/GenSync/Data/DatabaseContainer.h new file mode 100644 index 0000000..734f877 --- /dev/null +++ b/include/GenSync/Data/DatabaseContainer.h @@ -0,0 +1,74 @@ +/* This code is part of the GenSync project developed at Boston University. Please see the README for use and references. */ +// Created by GregoryFan on 7/10/2025 +// +#ifndef DATABASECONTAINER_H +#define DATABASECONTAINER_H +#include + +/** + * Implements a generic container that stores DataObject information. + * Information stored will be based on different database implementations. + */ +class DatabaseContainer : public DataContainer { + public: + //aliases + using DataContainer::iterator; + using DataContainer::const_iterator; + using DataContainer::size_type; + + /** + * Default destructor, meant to be overriden by subclasses. + */ + virtual ~DatabaseContainer() = default; + + /** + * @return An iterator that points to the beginning of the container. + */ + virtual iterator begin() override = 0; + + /** + * @return An iterator that points past the final item of the container. + */ + virtual iterator end() override = 0; + + /** + * @return A const iterator that points to the beginning of the container. + */ + virtual const_iterator begin() const override = 0; + + /** + * @return A const iterator that points to the final item of the container. + */ + virtual const_iterator end() const override = 0; + + /** + * @return The number of items inside the container. + */ + virtual size_type size() const override = 0; + + /** + * @return Whether the container has no DataObjects. + */ + virtual bool empty() const override = 0; + + /** + * Removes all DataObjects from the container. + */ + virtual void clear() override = 0; + + /** + * Removes the first DataObject that contain the internal data as the given DataObject. + * The internal data refers to the information that the DataObject represents. + * @param val The given DataObject. + * @return Returns true if object is successfully removed. + */ + virtual bool remove (const shared_ptr& val) override = 0; + + /** + * Pushes a given DataObject into the container for storage. + * @param val The given DataObject to store. + */ + virtual void add (const shared_ptr& val) override = 0; + +}; +#endif \ No newline at end of file diff --git a/include/GenSync/Data/InMemContainer.h b/include/GenSync/Data/InMemContainer.h index 1dfa605..a01e957 100644 --- a/include/GenSync/Data/InMemContainer.h +++ b/include/GenSync/Data/InMemContainer.h @@ -1,4 +1,7 @@ /* This code is part of the GenSync project developed at Boston University. Please see the README for use and references. */ +// Created by GregoryFan on 7/10/2025 +// + #ifndef INMEMCONTAINER_H #define INMEMCONTAINER_H #include diff --git a/include/GenSync/Data/SQLiteContainer.h b/include/GenSync/Data/SQLiteContainer.h new file mode 100644 index 0000000..6525f06 --- /dev/null +++ b/include/GenSync/Data/SQLiteContainer.h @@ -0,0 +1,180 @@ +/* This code is part of the GenSync project developed at Boston University. Please see the README for use and references. */ +// Created by GregoryFan on 7/10/2025 +// + +#ifdef USE_SQLITE +#ifndef SQLITE_CONTAINER_H +#define SQLITE_CONTAINER_H + +#include +#include + +class SQLiteContainer : public DatabaseContainer { + protected: + /** + * SQLite-based iterator for standard operation on an SQLite Table. + */ + class SQLiteIterator : public DataIterator { + public: + /** + * Constructs an iterator using a given database with + * a query on where to go. + */ + SQLiteIterator(sqlite3* db, const string& query); + + /** + * Constructs a default iterator with no given database. + */ + SQLiteIterator(); + + /** + * Default destructor that finalizes and closes its statement. + */ + ~SQLiteIterator(); + + /** + * @return A DataObject pointer that is reconstructed from the data the iterator is pointing at. + */ + shared_ptr operator*() const override; + + /** + * Moves the iterator up in the container's memory. + * Returns the iterator after it has moved positions. + * @return The iterator after it has moved positions in the container. + */ + DataIterator& operator++() override; + + /** + * Compares two iterators. + * Returns true if they fulfill three conditions: + * the other iterator is of type SQLiteContainer, + * both iterators have statements that point to the same database and table, + * and that both iterators are on the same row. + * @return Whether the two iterators point to the same information in a database and table. + */ + bool operator==(const DataIterator& other) const override; + + /** + * Compares two iterators. + * Returns false if they fulfill three conditions: + * the other iterator is of type SQLiteContainer, + * both iterators have statements that point to the same database and table, + * and that both iterators are on the same row. + * @return Whether the two iterators do not point to the same information in a database and table. + */ + bool operator!=(const DataIterator& other) const override; + + /** + * Creates a new iterator that points to the same object in the container. + * The iterator is destroyed when the pointer referring to it is moved or destroyed. + * @return A new iterator that is identical to this one. + */ + unique_ptr clone() const override; + + private: + /** + * The SQLite database the iterator refers to. + */ + sqlite3* _db; + + /** + * The SQLite statement the iterator is following. + */ + sqlite3_stmt* _stmt; + + /** + * The SQLite query that the iterator transforms into a statement. + */ + string _query; + + /** + * The index of the current row the iterator is at. + */ + int _currentRow; + + /** + * Whether the iterator is done with its statement. + */ + bool _done; + + /** + * Moves the iterator up in the table, if possible. + */ + void step(); + }; + + public: + /** + * Constructs a SQLiteContainer with a given database reference and table name. + * The database and table are created if they do not exist. + * @param ref The reference to the database by name. + * @param tableName The name of the table. + */ + SQLiteContainer(const string& ref, const string& tableName = "defaultTable"); + + /** + * Default Constructor. Closes the database if it exists. + */ + ~SQLiteContainer(); + + /** + * @return An iterator that points to the beginning of the container. + */ + iterator begin() override; + + /** + * @return An iterator that has finished and has no statement running. + * This iterator is meant the mark the end state of an iterator. + */ + iterator end() override; + + /** + * @return An const iterator that points to the beginning of the container. + */ + const_iterator begin() const override; + + /** + * @return A const iterator that has finished and has no statement running. + * This iterator is meant the mark the end state of an iterator. + */ + const_iterator end() const override; + + /** + * @return The number of items inside the table. + */ + size_type size() const override; + + /** + * @return Whether the table has no items. + */ + bool empty() const override; + + /** + * Removes all items from the table. + */ + void clear() override; + + /** + * Removes the first entry that contain the internal data as the given DataObject. + * The internal data refers to the information that the DataObject represents. + * @param val The given DataObject. + * @return Returns true if object is successfully removed. + */ + bool remove(const shared_ptr& val) override; + + /** + * Adds an entry with the internal data as the given DataObject. + * The internal data refers to the information that the DataObject represents. + * @param val The given DataObject. + */ + void add(const shared_ptr& val) override; + + private: + /** The table name of the SQLite table the container will read and write to.*/ + string tableName; + + /** The pointer to the SQLite database.*/ + sqlite3* db; +}; +#endif //SQLITE_CONTAINER_H +#endif \ No newline at end of file diff --git a/include/GenSync/Syncs/GenSync.h b/include/GenSync/Syncs/GenSync.h index df27306..ee8ab76 100644 --- a/include/GenSync/Syncs/GenSync.h +++ b/include/GenSync/Syncs/GenSync.h @@ -17,6 +17,10 @@ #include #include +#ifdef USE_SQLITE +#include +#endif + // namespace info using std::string; using std::cout; @@ -341,6 +345,11 @@ class GenSync { END // one after the end of iterable options }; + enum ContainerType { + UNDEFINED, + SQLite, + }; + private: @@ -350,6 +359,44 @@ class GenSync { */ GenSync(); + /** + * Private GenSync constructor, for setting myData to a database type and whether to clear it when the destructor is called. + * @param cVec The vector of other GenSync's with whom this data structure might + * be synchronized. + * @param mVec The vector of synchronization methods that this GenSync + * should be prepared to use. The order of these methods + * is significant. + * @param dataCont The type of container the GenSync object will use to hold information. + * @param cleanData Signifies whether the GenSync object will clear the database when constructed. + * @param postProcessing Cleanup functions to call after processing has been completed. The default + * is to incorporate all items from the remote agent into my own data. + * The parameters of postProcessing are: + * 0. otherMinusSelf - what the remote host has that I do not + * 1. myData - all my data + * 2. add - a pointer to the add method of my GenSync object + * 3. del - a pointer to the dell method of my GenSync object + * 4. pGenSync - a pointer tox this GenSync object + * @param data The initial data with which to populate the data structure. The data is added element by element + * so that synchronization method metadata can be properly maintained. Initilizes to the empty list + * if not specified. + * + */ + GenSync( + const vector> &cVec, + const vector> &mVec, + const shared_ptr dataCont, + const bool cleanData = false, + void (*postProcessing)( + list>, + const DataContainer&, + void (GenSync::*add)(shared_ptr), + bool (GenSync::*del)(shared_ptr), + GenSync *pGenSync) + = SyncMethod::postProcessing_SET, + const list> &data = list>() + + ); + /** A pointer to the postprocessing function **/ void (*_PostProcessing)(list>, const DataContainer&, void (GenSync::*add)(shared_ptr), bool (GenSync::*del)(shared_ptr), GenSync *pGenSync){}; @@ -417,6 +464,27 @@ class GenSync::Builder { return *this; } + /** + * Sets the container to a database type. + * @param theContainer The container type. + * @param dbRef A string reference to the database name. + * @param tableName A string reference to the table name. + * @param clearDBData Whether to clear the data when the destructor is called. + */ + Builder& setContainer(ContainerType theContainer, const string& dbRef, const string& tableName = "defaultTable", bool clearDBData = false){ + clearData = clearDBData; + switch(theContainer){ + #ifdef USE_SQLITE + case ContainerType::SQLite: + this -> dataCont = Nullable>(make_shared(dbRef, tableName)); + break; + #endif + default: + throw invalid_argument("Unknown Container Type"); + } + return *this; + } + /** * Sets the host to which to connect for synchronization in a socket-based sync. */ @@ -612,6 +680,8 @@ class GenSync::Builder { Nullable> probMatrix; /** Probability matrix for element types in MET */ Nullable> cellTypeFunc; /** Function which outputs size of cell type given cell type index for MET */ Nullable(size_t)>> degMatrixFunc; /** Function which outputs degrees of cell type given cell type index for MET */ + Nullable> dataCont; + bool clearData; // ... bookkeeping variables diff --git a/src/Data/InMemContainer.cpp b/src/Data/InMemContainer.cpp index 8ccd47b..8f4dcdd 100644 --- a/src/Data/InMemContainer.cpp +++ b/src/Data/InMemContainer.cpp @@ -1,3 +1,6 @@ +// +// Created by GregoryFan on 7/10/2025 +// #include InMemContainer::~InMemContainer(){clear();} diff --git a/src/Data/SQLiteContainer.cpp b/src/Data/SQLiteContainer.cpp new file mode 100644 index 0000000..8f0e2fa --- /dev/null +++ b/src/Data/SQLiteContainer.cpp @@ -0,0 +1,243 @@ +// +// Created by GregoryFan on 7/10/2025 +// + +#ifdef USE_SQLITE +#include + +//SQLiteIterator Functions + +SQLiteContainer::SQLiteIterator::SQLiteIterator(sqlite3* db, const string& query): _query(query), _currentRow(0), _db(db), _done(true), _stmt(nullptr){ + //Prepares the Iterator with a given database and query. + if (sqlite3_prepare_v2(_db, query.c_str(), -1, &_stmt, nullptr) != SQLITE_OK) { + throw std::runtime_error("Failed to prepare statement"); + } + //Enters the first element. + step(); +} + +SQLiteContainer::SQLiteIterator::SQLiteIterator(): _db(nullptr), _stmt(nullptr), _done(true){} + +SQLiteContainer::SQLiteIterator::~SQLiteIterator() { + if (_stmt) sqlite3_finalize(_stmt); +} + +shared_ptr SQLiteContainer::SQLiteIterator::operator*() const{ + //Trying to dereference end iterator + if (_done) throw std::runtime_error("Dereferencing end iterator"); + + //Extracts data and creates DataObject from it. + const void* blob = sqlite3_column_blob(_stmt, 1); + int size = sqlite3_column_bytes(_stmt, 1); + string data(reinterpret_cast(blob), size); + return make_shared(data); +} + +SQLiteContainer::DataIterator& SQLiteContainer::SQLiteIterator::operator++(){ + step(); + return *this; +} + +bool SQLiteContainer::SQLiteIterator::operator==(const DataIterator& other) const{ + //Ensures other is a SQLiteIterator + auto o = dynamic_cast(&other); + if (!o) return false; + + //Compares by doneness or by having the same row and statement. + return _done == o->_done || (_stmt == o->_stmt && _currentRow == o->_currentRow); +} + +bool SQLiteContainer::SQLiteIterator::operator!=(const DataIterator& other) const{ + return !(*this == other); +} + +unique_ptr SQLiteContainer::SQLiteIterator::clone() const{ + auto cloned = unique_ptr(new SQLiteIterator(_db, _query)); + + // Step to same position + for (int i = 1; i < _currentRow; ++i) { + cloned->step(); + } + + return cloned; +} + +void SQLiteContainer::SQLiteIterator::step(){ + //Statement is not set up. + if (!_stmt) return; + + //Goes onto the next item in the table. + int rc = sqlite3_step(_stmt); + if (rc == SQLITE_ROW) { + //More elements to go through + _done = false; + ++_currentRow; + } else if (rc == SQLITE_DONE) { + //No more elements to go through + _done = true; + sqlite3_finalize(_stmt); + _stmt = nullptr; + } else { + throw std::runtime_error("SQLite step error"); + } +} + + +//SQLiteContainer Functions + +SQLiteContainer::SQLiteContainer(const string& ref, const string& table) : tableName(table) { + //Opens the database if possible. + if (sqlite3_open(ref.c_str(), &db) != SQLITE_OK) { + std::cerr << "Failed to open database: " << sqlite3_errmsg(db) << std::endl; + db = nullptr; + return; + } + + //Creates a table if needed. + string statement = + "CREATE TABLE IF NOT EXISTS " + tableName + " (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + "data BLOB);"; + + char* errMsg = nullptr; + if (sqlite3_exec(db, statement.c_str(), nullptr, nullptr, &errMsg) != SQLITE_OK) { + std::cerr << "Failed to create table: " << errMsg << std::endl; + sqlite3_free(errMsg); + sqlite3_close(db); + db = nullptr; + } +} + +SQLiteContainer::~SQLiteContainer(){ + if (db) { + sqlite3_close(db); + } +} + +//Methods +SQLiteContainer::iterator SQLiteContainer::begin(){ + //Takes every item from the list for the iterator to go through. + string query = "SELECT * FROM " + tableName; + return unique_ptr(new SQLiteIterator(db, query)); +} + +SQLiteContainer::iterator SQLiteContainer::end(){ + //Sends a blank iterator to denote the end of a iterator statement. + return unique_ptr(new SQLiteIterator()); +} + +SQLiteContainer::const_iterator SQLiteContainer::begin() const{ + //Takes every item from the list for the iterator to go through. + string query = "SELECT * FROM " + tableName; + return unique_ptr(new SQLiteIterator(db, query)); +} + +SQLiteContainer::const_iterator SQLiteContainer::end() const{ + //Sends a blank iterator to denote the end of a iterator statement. + return unique_ptr(new SQLiteIterator()); +} + +DataContainer::size_type SQLiteContainer::size() const{ + sqlite3_stmt* stmt = nullptr; + string query = "SELECT COUNT(*) FROM " + tableName; + DataContainer::size_type count = 0; + + if (sqlite3_prepare_v2(db, query.c_str(), -1, &stmt, nullptr) != SQLITE_OK) { + throw std::runtime_error("Failed to prepare COUNT query: " + std::string(sqlite3_errmsg(db))); + } + + if (sqlite3_step(stmt) == SQLITE_ROW) { + count = static_cast(sqlite3_column_int(stmt, 0)); + } else { + sqlite3_finalize(stmt); + throw std::runtime_error("Failed to execute COUNT query: " + std::string(sqlite3_errmsg(db))); + } + + sqlite3_finalize(stmt); + return count; +} + +bool SQLiteContainer::empty() const{ + //Takes a single element to check if it is empty. + sqlite3_stmt* stmt; + string sql = "SELECT 1 FROM " + tableName + " LIMIT 1;"; + + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) { + throw std::runtime_error("Failed to prepare empty check: " + std::string(sqlite3_errmsg(db))); + } + + int rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + + return rc != SQLITE_ROW; +} + +void SQLiteContainer::clear(){ + sqlite3_stmt* stmt; + string sql = "DELETE FROM " + tableName + ";"; + + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) { + throw std::runtime_error("Failed to prepare clear: " + std::string(sqlite3_errmsg(db))); + } + + if (sqlite3_step(stmt) != SQLITE_DONE) { + sqlite3_finalize(stmt); + throw std::runtime_error("Failed to clear table: " + std::string(sqlite3_errmsg(db))); + } + + sqlite3_finalize(stmt); +} + +bool SQLiteContainer::remove(const shared_ptr& val){ + sqlite3_stmt* stmt; + int sizeBefore = size(); + string data = val->to_string(); + string statement = "DELETE FROM " + tableName + " WHERE data=? LIMIT 1"; + + if (sqlite3_prepare_v2(db, statement.c_str(), -1, &stmt, nullptr) != SQLITE_OK) { + throw std::runtime_error("Failed to prepare statement: " + std::string(sqlite3_errmsg(db))); + } + + if (sqlite3_bind_blob(stmt, 1, val->to_string().data(), static_cast(val->to_string().size()), SQLITE_TRANSIENT) != SQLITE_OK) { + sqlite3_finalize(stmt); + throw std::runtime_error("Failed to bind id"); + } + + if (sqlite3_step(stmt) != SQLITE_DONE) { + sqlite3_finalize(stmt); + throw std::runtime_error("Failed to execute delete"); + } + + sqlite3_finalize(stmt); + + return sizeBefore < size(); +} + +void SQLiteContainer::add(const shared_ptr& val){ + sqlite3_stmt* stmt; + string statement = "INSERT INTO " + tableName + " (data) VALUES (?);"; + + if (sqlite3_prepare_v2(db, statement.c_str(), -1, &stmt, nullptr) != SQLITE_OK) { + throw std::runtime_error("Failed to prepare statement: " + std::string(sqlite3_errmsg(db))); + } + + if (sqlite3_bind_blob(stmt, 1, val->to_string().data(), static_cast(val->to_string().size()), SQLITE_TRANSIENT) != SQLITE_OK) { + sqlite3_finalize(stmt); + throw std::runtime_error("Failed to bind data"); + } + + // Execute the statement + if (sqlite3_step(stmt) != SQLITE_DONE) { + string err = sqlite3_errmsg(db); + sqlite3_finalize(stmt); + throw std::runtime_error("Failed to execute insert" + err); + } + + // Get the auto-generated ID + int inserted_id = static_cast(sqlite3_last_insert_rowid(db)); + val->setObjectID(inserted_id); + + // Finalize to free the statement object + sqlite3_finalize(stmt); +} +#endif diff --git a/src/Syncs/GenSync.cpp b/src/Syncs/GenSync.cpp index c0913d8..85a9f44 100644 --- a/src/Syncs/GenSync.cpp +++ b/src/Syncs/GenSync.cpp @@ -77,6 +77,29 @@ GenSync::GenSync(const vector> &cVec, const vector> &cVec, + const vector> &mVec, + const shared_ptr dataCont, + const bool cleanData, + void (*postProcessing)(list>, const DataContainer& ,void (GenSync::*add)(shared_ptr),bool (GenSync::*del)(shared_ptr), GenSync *pGenSync), + const list> &data + ) +{ + myData = dataCont; + myCommVec = cVec; + mySyncVec = mVec; + outFile = nullptr; // no output file is being used + _PostProcessing = postProcessing; + if(cleanData){ + myData -> clear(); + } + // add each datum one by one + auto itData = data.begin(); + for (; itData != data.end(); itData++) + addElem(*itData); +} + // destruct a gensync object GenSync::~GenSync() { @@ -529,10 +552,18 @@ GenSync GenSync::Builder::build() { } theMeths.push_back(myMeth); - if (fileName.isNullQ()) // is data to be drawn from a file? - return GenSync(theComms, theMeths, _postProcess); + GenSync g; + + if (this->dataCont.isNullQ()){ + this -> dataCont = Nullable>(make_shared()); + } + + if (fileName.isNullQ()) + g = GenSync(theComms, theMeths, *this->dataCont, clearData, _postProcess); else - return GenSync(theComms, theMeths, fileName); + g = GenSync(theComms, theMeths, fileName); + + return g; } // static consts diff --git a/src/TryMe.cpp b/src/TryMe.cpp index a46637a..192565f 100644 --- a/src/TryMe.cpp +++ b/src/TryMe.cpp @@ -2,45 +2,46 @@ #include int main() { - // BUILD the first host - GenSync host1 = GenSync::Builder(). - setProtocol(GenSync::SyncProtocol::CPISync). // CPISync protocol - setComm(GenSync::SyncComm::socket). // communicate over network sockets - setMbar(5). // required parameter for CPISync - build(); - - // BUILD the second host - GenSync host2 = GenSync::Builder(). - setProtocol(GenSync::SyncProtocol::CPISync). - setComm(GenSync::SyncComm::socket). - setMbar(5). - build(); - - // ADD elements to each host - // ... host 1 - host1.addElem(make_shared('a')); // DataObject containing a character 'a' - host1.addElem(make_shared('b')); - host1.addElem(make_shared('c')); - - // ... host 2 - host2.addElem(make_shared('b')); - host2.addElem(make_shared('d')); - // FORK into two processes if (fork()) { + // BUILD the first host + GenSync host1 = GenSync::Builder(). + setProtocol(GenSync::SyncProtocol::CPISync). // CPISync protocol + setComm(GenSync::SyncComm::socket). // communicate over network sockets + setContainer(GenSync::ContainerType::SQLite, "temp", "temp", true). + setMbar(5). // required parameter for CPISync + build(); + + // ... host 1 + host1.addElem(make_shared('a')); // DataObject containing a character 'a' + host1.addElem(make_shared('b')); + host1.addElem(make_shared('c')); + // ... PARENT process host1.clientSyncBegin(0); // set up the 0-th synchronizer and connect to a server cout << "host 1 now has "; for (auto &i: host1.dumpElements()) // print out the elements at host 1 - cout << i << " "; + cout << i << " "; cout << endl; } else { + // BUILD the second host + GenSync host2 = GenSync::Builder(). + setProtocol(GenSync::SyncProtocol::CPISync). + setComm(GenSync::SyncComm::socket). + setContainer(GenSync::ContainerType::SQLite, "temp2", "temp2", true). + setMbar(5). + build(); + + // ... host 2 + host2.addElem(make_shared('b')); + host2.addElem(make_shared('d')); + // ... CHILD process host2.serverSyncBegin(0); // set up the 0-th synchronizer and wait for connections cout << "host 2 now has "; for (auto &i: host2.dumpElements()) // print out the elements at host 2 - cout << i << " "; + cout << i << " "; cout << endl; } diff --git a/testDB b/testDB new file mode 100644 index 0000000..e69de29 diff --git a/tests/TestAuxiliary.h b/tests/TestAuxiliary.h index cd02e44..b3ccea1 100644 --- a/tests/TestAuxiliary.h +++ b/tests/TestAuxiliary.h @@ -38,6 +38,12 @@ const int numParts = 3; // Partitions per level for divide-and-conquer syncs const int numExpElem = UCHAR_MAX*4; // Max elements in an IBLT for IBLT syncs const size_t largeNumExpElems = largeLimit * 3; // Maximum sum of CLIENT_MINUS_SERVER and SEVER_MINUS_CLIENT and SIMILAR +//DataContainer Testing +const int TEST_ITER = 50; //Number of Test Iterations +const int CONTAINERSIZE = 20; //Default Size of Container +const int LOWER_BOUND_SIZE = 20; //Minimum Length of Data +const int UPPER_BOUND_SIZE = 50; //Maximum Length of Data + // helpers /** @@ -1297,4 +1303,133 @@ inline bool socketSendReceiveTest(){ return true; } + +/** + * This function tests a DataContainer by adding elements, + * then iterates through it to verify the elements have been properly added + * by comparing it with a list data structure. + * @return true iff all the test succeeds. + */ +inline bool dataContainerAddTest(DataContainer& container){ + //Create container + list> objList; + for(int ii = 0; ii < TEST_ITER; ii++){ + //Fill containers + for(int jj = 0; jj < CONTAINERSIZE; jj++){ + shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); + container.add(obj); + objList.push_back(obj); + } + + auto listIt = objList.begin(); + auto contIt = container.begin(); + + //Iterates through both lists + //Ensures the InMemoryContainer has + int count = 0; + for(; contIt != container.end(); contIt++){ + if(!(*(*contIt) == *(*listIt))) return false; + listIt++; + } + } + return true; +} + +/** + * This function tests a DataContainer's clear function + * by adding elements and checking that it is empty after it is cleared. + * @return true iff all the test succeeds. + */ +inline bool dataContainerClearTest(DataContainer& container){ + for(int ii = 0; ii < TEST_ITER; ii++){ + //Fills container + shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); + container.add(obj); + container.clear(); + //Checks if container is empty after clear + if(!container.empty() || container.size() != 0) return false; + } + return true; +} + +/** + * This function tests a DataContainer's empty function + * by checking if it is empty before and after elements are added. + * @return true iff all the test succeeds. + */ +inline bool dataContainerEmptyTest(DataContainer& container){ + for(int ii = 0; ii < TEST_ITER; ii++){ + //Checks if container is empty when it is empty + if(!container.empty()) return false; + shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); + container.add(obj); + //Checks if container is empty when it is not empty + if(container.empty()) return false; + container.clear(); + } + return true; +} + +/** + * This function tests a DataContainer's size function + * by adding a random amount of elements and verifies + * its sizes is equal to the expected amount. + * @return true iff all the test succeeds. + */ +inline bool dataContainerSizeTest(DataContainer& container){ + for(int ii = 0; ii < TEST_ITER; ii++){ + //Determines a random number between 0 to 99. + int randSize = rand() % 100; + //Fills the loop that amount of times + for(int jj = 0; jj < randSize; jj++){ + shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); + container.add(obj); + } + //Compares the two sizes to ensure they are the same. + if(static_cast(randSize) != container.size()) return false; + container.clear(); + } + return true; +} + +/** + * This function tests a DataContainer's remove function + * by adding elements then removing half of them. + * Then it's elements are compared with a list whose elements have + * not been removed. + * @return true iff all the test succeeds. + */ +inline bool dataContainerRemoveTest(DataContainer& container){ + list> objList; + + for(int ii = 0; ii < TEST_ITER; ii++){ + //Fill containers + for(int jj = 0; jj < CONTAINERSIZE; jj++){ + shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); + container.add(obj); + objList.push_back(obj); + } + + auto listIt = objList.begin(); + auto contIt = container.begin(); + + //Remove half the list from the In Memory Container + for(int jj = 0; jj < CONTAINERSIZE/2; jj++){ + container.remove(*listIt); + listIt++; + } + + contIt = container.begin(); + + //Compares the resultant container against the list + for(; contIt != container.end(); contIt++){ + if(!(*(*contIt) == *(*listIt))) return false; + listIt++; + } + container.clear(); + objList.clear(); + } + return true; +} + #endif //CPISYNCLIB_GENERIC_SYNC_TESTS_H diff --git a/tests/unit/InMemContainerTest.cpp b/tests/unit/InMemContainerTest.cpp index cc8f062..103b4e9 100644 --- a/tests/unit/InMemContainerTest.cpp +++ b/tests/unit/InMemContainerTest.cpp @@ -10,106 +10,26 @@ void InMemContainerTest::setUp(){ } void InMemContainerTest::addTest(){ - //Create containers - list> objList; InMemContainer container; - - for(int ii = 0; ii < TEST_ITER; ii++){ - //Fill containers - for(int jj = 0; jj < CONTAINERSIZE; jj++){ - shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); - container.add(obj); - objList.push_back(obj); - } - - auto listIt = objList.begin(); - auto contIt = container.begin(); - - //Iterates through both lists - //Ensures the InMemoryContainer has - for(; contIt != container.end(); contIt++){ - CPPUNIT_ASSERT_EQUAL(*(*contIt) , *(*listIt)); - listIt++; - } - } - CPPUNIT_ASSERT(true); + CPPUNIT_ASSERT(dataContainerAddTest(container)); } void InMemContainerTest::clearTest(){ InMemContainer container; - for(int ii = 0; ii < TEST_ITER; ii++){ - //Fills container - shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); - container.add(obj); - container.clear(); - //Checks if container is empty after clear - CPPUNIT_ASSERT(container.empty() && container.size() == 0); - } - CPPUNIT_ASSERT(true); + CPPUNIT_ASSERT(dataContainerClearTest(container)); } void InMemContainerTest::emptyTest(){ InMemContainer container; - for(int ii = 0; ii < TEST_ITER; ii++){ - //Checks if container is empty when it is empty - CPPUNIT_ASSERT_EQUAL(true, container.empty()); - shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); - container.add(obj); - //Checks if container is empty when it is not empty - CPPUNIT_ASSERT_EQUAL(false, container.empty()); - container.clear(); - } - CPPUNIT_ASSERT(true); + CPPUNIT_ASSERT(dataContainerEmptyTest(container)); } void InMemContainerTest::sizeTest(){ InMemContainer container; - for(int ii = 0; ii < TEST_ITER; ii++){ - //Determines a random number between 0 to 99. - int randSize = rand() % 100; - //Fills the loop that amount of times - for(int jj = 0; jj < randSize; jj++){ - shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); - container.add(obj); - } - //Compares the two sizes to ensure they are the same. - CPPUNIT_ASSERT_EQUAL(static_cast(randSize),container.size()); - container.clear(); - } - CPPUNIT_ASSERT(true); + CPPUNIT_ASSERT(dataContainerSizeTest(container)); } void InMemContainerTest::removeTest(){ - //Create containers - list> objList; InMemContainer container; - - for(int ii = 0; ii < TEST_ITER; ii++){ - //Fill containers - for(int jj = 0; jj < CONTAINERSIZE; jj++){ - shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); - container.add(obj); - objList.push_back(obj); - } - - auto listIt = objList.begin(); - auto contIt = container.begin(); - - //Remove half the list from the In Memory Container - for(int jj = 0; jj < CONTAINERSIZE/2; jj++){ - container.remove(*listIt); - listIt++; - } - - contIt = container.begin(); - - //Compares the resultant container against the list - for(; contIt != container.end(); contIt++){ - CPPUNIT_ASSERT_EQUAL(*(*contIt) , *(*listIt)); - listIt++; - } - container.clear(); - objList.clear(); - } - CPPUNIT_ASSERT(true); + CPPUNIT_ASSERT(dataContainerRemoveTest(container)); } \ No newline at end of file diff --git a/tests/unit/InMemContainerTest.h b/tests/unit/InMemContainerTest.h index 09fc438..c04560f 100644 --- a/tests/unit/InMemContainerTest.h +++ b/tests/unit/InMemContainerTest.h @@ -20,26 +20,6 @@ class InMemContainerTest : public CppUnit::TestFixture { */ const int SEED = 1029; - /** - * The amount of iteration each test goes through. - */ - const int TEST_ITER = 50; - - /** - * The size of the container by default. - */ - const int CONTAINERSIZE = 20; - - /** - * The minimum size for a random string assigned as data to a DataObject. - */ - const int LOWER_BOUND_SIZE = 20; - - /** - * The maximum size for a random string assigned as data to a DataObject. - */ - const int UPPER_BOUND_SIZE = 50; - /** * Sets up the random seed. */ diff --git a/tests/unit/SQLiteContainerTest.cpp b/tests/unit/SQLiteContainerTest.cpp new file mode 100644 index 0000000..ecaa9b4 --- /dev/null +++ b/tests/unit/SQLiteContainerTest.cpp @@ -0,0 +1,47 @@ +// +// Created by GregoryFan on 7/10/2025 +// + +#ifdef USE_SQLITE +#include +#include "SQLiteContainerTest.h" + +CPPUNIT_TEST_SUITE_REGISTRATION(SQLiteContainerTest); + +//Sets the random seed +void SQLiteContainerTest::setUp(){ + srand(SEED); + container = unique_ptr(new SQLiteContainer("testDB", "testTable")); +} + +void SQLiteContainerTest::tearDown(){ + container->clear(); + remove("testDB"); +} + + +void SQLiteContainerTest::addTest(){ + //SQLiteContainer container("testDB"); + CPPUNIT_ASSERT(dataContainerAddTest(*container)); +} + +void SQLiteContainerTest::clearTest(){ + //SQLiteContainer container("testDB"); + CPPUNIT_ASSERT(dataContainerClearTest(*container)); +} + +void SQLiteContainerTest::emptyTest(){ + //SQLiteContainer container("testDB"); + CPPUNIT_ASSERT(dataContainerEmptyTest(*container)); +} + +void SQLiteContainerTest::sizeTest(){ + //SQLiteContainer container("testDB"); + CPPUNIT_ASSERT(dataContainerSizeTest(*container)); +} + +void SQLiteContainerTest::removeTest(){ + //SQLiteContainer container("testDB"); + CPPUNIT_ASSERT(dataContainerRemoveTest(*container)); +} +#endif \ No newline at end of file diff --git a/tests/unit/SQLiteContainerTest.h b/tests/unit/SQLiteContainerTest.h new file mode 100644 index 0000000..0a4fb1a --- /dev/null +++ b/tests/unit/SQLiteContainerTest.h @@ -0,0 +1,59 @@ +/* This code is part of the GenSync project developed at Boston University. Please see the README for use and references. */ +// Created by GregoryFan on 7/10/2025 +// + +#ifdef USE_SQLITE +#ifndef SQLITE_CONTAINER_TEST_H +#define SQLITE_CONTAINER_TEST_H +#include +#include +#include "../TestAuxiliary.h" + +class SQLiteContainerTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(SQLiteContainerTest); + CPPUNIT_TEST(addTest); + CPPUNIT_TEST(clearTest); + CPPUNIT_TEST(emptyTest); + CPPUNIT_TEST(sizeTest); + CPPUNIT_TEST(removeTest); + CPPUNIT_TEST_SUITE_END(); + + public: + /** + * The static seed the tests are set to. + */ + const int SEED = 1920; + void setUp() override; + void tearDown() override; + + private: + unique_ptr container; + /** + * Adds elements to a container and iterates through them using a list + * as comparison to ensure values have been successfully inserted. + */ + void addTest(); + + /** + * Tests the clear function by repeatedly filling and clearing the container. + */ + void clearTest(); + + /** + * Tests the empty function by repeatedly checking when the container is filled and when it is cleared. + */ + void emptyTest(); + + /** + * Repeatedly fills a container to a random size and compares its size value to the expected amount. + */ + void sizeTest(); + + /** + * First fills a container, then removes half the elements and compares the other half to a list. + */ + void removeTest(); + +}; +#endif //SQLITE_CONTAINER_TEST_H +#endif \ No newline at end of file From faaadc00e3c1ca524206288402bd1a8069ff405b Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Mon, 28 Jul 2025 11:49:36 -0400 Subject: [PATCH 4/9] Removed Excess FIle --- testDB | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 testDB diff --git a/testDB b/testDB deleted file mode 100644 index e69de29..0000000 From 976f020015987cea716ee6492958d14edd38d8c7 Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Mon, 28 Jul 2025 11:59:33 -0400 Subject: [PATCH 5/9] Removed testing code from TryMe --- src/TryMe.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/TryMe.cpp b/src/TryMe.cpp index 192565f..c101147 100644 --- a/src/TryMe.cpp +++ b/src/TryMe.cpp @@ -8,7 +8,6 @@ int main() { GenSync host1 = GenSync::Builder(). setProtocol(GenSync::SyncProtocol::CPISync). // CPISync protocol setComm(GenSync::SyncComm::socket). // communicate over network sockets - setContainer(GenSync::ContainerType::SQLite, "temp", "temp", true). setMbar(5). // required parameter for CPISync build(); @@ -29,7 +28,6 @@ int main() { GenSync host2 = GenSync::Builder(). setProtocol(GenSync::SyncProtocol::CPISync). setComm(GenSync::SyncComm::socket). - setContainer(GenSync::ContainerType::SQLite, "temp2", "temp2", true). setMbar(5). build(); From 0615f05660b34fa6440a2d7671ac581082efb135 Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Mon, 4 Aug 2025 11:20:44 -0400 Subject: [PATCH 6/9] Additional Documentation --- include/GenSync/Aux/UID.h | 3 +++ include/GenSync/Data/DatabaseContainer.h | 4 ++-- include/GenSync/Syncs/GenSync.h | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/GenSync/Aux/UID.h b/include/GenSync/Aux/UID.h index e75c5c8..dcc57a9 100644 --- a/include/GenSync/Aux/UID.h +++ b/include/GenSync/Aux/UID.h @@ -31,6 +31,9 @@ class UID { return myID; } + /** + * @return Sets the ID of this object. + */ void setObjectID(int newID){ myID = newID; } diff --git a/include/GenSync/Data/DatabaseContainer.h b/include/GenSync/Data/DatabaseContainer.h index 734f877..c7ea5b4 100644 --- a/include/GenSync/Data/DatabaseContainer.h +++ b/include/GenSync/Data/DatabaseContainer.h @@ -27,7 +27,7 @@ class DatabaseContainer : public DataContainer { virtual iterator begin() override = 0; /** - * @return An iterator that points past the final item of the container. + * @return An iterator that points to the end of the container. */ virtual iterator end() override = 0; @@ -37,7 +37,7 @@ class DatabaseContainer : public DataContainer { virtual const_iterator begin() const override = 0; /** - * @return A const iterator that points to the final item of the container. + * @return A const iterator that points to the end of the container. */ virtual const_iterator end() const override = 0; diff --git a/include/GenSync/Syncs/GenSync.h b/include/GenSync/Syncs/GenSync.h index ee8ab76..560e6f4 100644 --- a/include/GenSync/Syncs/GenSync.h +++ b/include/GenSync/Syncs/GenSync.h @@ -345,8 +345,11 @@ class GenSync { END // one after the end of iterable options }; + /** + * Container types that are used for database storage. + * This does not check for whether the database library is enabled. + */ enum ContainerType { - UNDEFINED, SQLite, }; From a0a0c39437efdfc0d9169adfed048b190695f95f Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Wed, 6 Aug 2025 15:43:56 -0400 Subject: [PATCH 7/9] Added Batching For Time Effeciency --- include/GenSync/Data/DataContainer.h | 1 - include/GenSync/Data/SQLiteContainer.h | 47 +++++++++++++++++++++++--- include/GenSync/Syncs/GenSync.h | 5 +-- src/TryMe.cpp | 2 ++ tests/TestAuxiliary.h | 1 + tests/unit/SQLiteContainerTest.cpp | 4 +-- 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/include/GenSync/Data/DataContainer.h b/include/GenSync/Data/DataContainer.h index 16ecdbe..c222f8e 100644 --- a/include/GenSync/Data/DataContainer.h +++ b/include/GenSync/Data/DataContainer.h @@ -151,7 +151,6 @@ class DataContainer{ using iterator = DataIteratorWrapper; using const_iterator = DataIteratorWrapper; using size_type = list>::size_type; - /** * Default destructor, meant to be overriden by subclasses. */ diff --git a/include/GenSync/Data/SQLiteContainer.h b/include/GenSync/Data/SQLiteContainer.h index 6525f06..6fc01b6 100644 --- a/include/GenSync/Data/SQLiteContainer.h +++ b/include/GenSync/Data/SQLiteContainer.h @@ -2,13 +2,17 @@ // Created by GregoryFan on 7/10/2025 // -#ifdef USE_SQLITE +//#ifdef USE_SQLITE #ifndef SQLITE_CONTAINER_H #define SQLITE_CONTAINER_H #include #include +/** + * Implements a Data Container that uses SQLite to store data. + * This container is not thread safe. + */ class SQLiteContainer : public DatabaseContainer { protected: /** @@ -107,13 +111,15 @@ class SQLiteContainer : public DatabaseContainer { /** * Constructs a SQLiteContainer with a given database reference and table name. * The database and table are created if they do not exist. + * Also allows for a custom buffer size for when commands are executed to the disk. * @param ref The reference to the database by name. * @param tableName The name of the table. + * @param batchSize The size of the buffer before operations are committed to disk. */ - SQLiteContainer(const string& ref, const string& tableName = "defaultTable"); + SQLiteContainer(const string& ref, const string& tableName = "defaultTable", int batchSize = 100); /** - * Default Constructor. Closes the database if it exists. + * Default Constructor. Commits all commands and closes database if possible. */ ~SQLiteContainer(); @@ -150,7 +156,8 @@ class SQLiteContainer : public DatabaseContainer { bool empty() const override; /** - * Removes all items from the table. + * Removes all items from the table. + * Doing this commits all commands in the buffer. */ void clear() override; @@ -169,12 +176,42 @@ class SQLiteContainer : public DatabaseContainer { */ void add(const shared_ptr& val) override; + /** + * Sets the number of operations to buffer before automatically committing them to disk. + */ + void setBatchSize(int newSize); + private: /** The table name of the SQLite table the container will read and write to.*/ string tableName; /** The pointer to the SQLite database.*/ sqlite3* db; + + /** + * Whether or not a buffer is active. + */ + bool transactionInProgress = false; + + /** + * The number of operations that has been put in the buffer. + */ + int operationCount = 0; + + /** + * The maximum number of operations before they are executed to the disk. + */ + int BATCH_SIZE = 100; + + /** + * Begins a buffer that stores given commands to the disk. + */ + void beginTransaction(); + + /** + * Executs all commands within the buffer to the disk. + */ + void commitTransaction(); }; #endif //SQLITE_CONTAINER_H -#endif \ No newline at end of file +//#endif \ No newline at end of file diff --git a/include/GenSync/Syncs/GenSync.h b/include/GenSync/Syncs/GenSync.h index 560e6f4..f8ea58c 100644 --- a/include/GenSync/Syncs/GenSync.h +++ b/include/GenSync/Syncs/GenSync.h @@ -473,13 +473,14 @@ class GenSync::Builder { * @param dbRef A string reference to the database name. * @param tableName A string reference to the table name. * @param clearDBData Whether to clear the data when the destructor is called. + * @param batchSize The size of how many commands the database holds before executing it to the disk. */ - Builder& setContainer(ContainerType theContainer, const string& dbRef, const string& tableName = "defaultTable", bool clearDBData = false){ + Builder& setContainer(ContainerType theContainer, const string& dbRef, const string& tableName = "defaultTable", bool clearDBData = false, int batchSize = 100){ clearData = clearDBData; switch(theContainer){ #ifdef USE_SQLITE case ContainerType::SQLite: - this -> dataCont = Nullable>(make_shared(dbRef, tableName)); + this -> dataCont = Nullable>(make_shared(dbRef, tableName, batchSize)); break; #endif default: diff --git a/src/TryMe.cpp b/src/TryMe.cpp index c101147..b6c74da 100644 --- a/src/TryMe.cpp +++ b/src/TryMe.cpp @@ -9,6 +9,7 @@ int main() { setProtocol(GenSync::SyncProtocol::CPISync). // CPISync protocol setComm(GenSync::SyncComm::socket). // communicate over network sockets setMbar(5). // required parameter for CPISync + setContainer(GenSync::ContainerType::SQLite, "testDB", "testTable", true). build(); // ... host 1 @@ -28,6 +29,7 @@ int main() { GenSync host2 = GenSync::Builder(). setProtocol(GenSync::SyncProtocol::CPISync). setComm(GenSync::SyncComm::socket). + setContainer(GenSync::ContainerType::SQLite, "testDB2", "testTable2", true). setMbar(5). build(); diff --git a/tests/TestAuxiliary.h b/tests/TestAuxiliary.h index b3ccea1..61f9545 100644 --- a/tests/TestAuxiliary.h +++ b/tests/TestAuxiliary.h @@ -1319,6 +1319,7 @@ inline bool dataContainerAddTest(DataContainer& container){ shared_ptr obj = make_shared(randString(LOWER_BOUND_SIZE, UPPER_BOUND_SIZE)); container.add(obj); objList.push_back(obj); + } auto listIt = objList.begin(); diff --git a/tests/unit/SQLiteContainerTest.cpp b/tests/unit/SQLiteContainerTest.cpp index ecaa9b4..1bb6935 100644 --- a/tests/unit/SQLiteContainerTest.cpp +++ b/tests/unit/SQLiteContainerTest.cpp @@ -11,12 +11,12 @@ CPPUNIT_TEST_SUITE_REGISTRATION(SQLiteContainerTest); //Sets the random seed void SQLiteContainerTest::setUp(){ srand(SEED); - container = unique_ptr(new SQLiteContainer("testDB", "testTable")); + container = unique_ptr(new SQLiteContainer("testDB2", "testTable2")); } void SQLiteContainerTest::tearDown(){ container->clear(); - remove("testDB"); + remove("testDB2"); } From fdb61d339e0ff60fde022bbbcee113c4a0d571d1 Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Wed, 6 Aug 2025 15:44:34 -0400 Subject: [PATCH 8/9] Included Updates for SQLiteContainer --- src/Data/SQLiteContainer.cpp | 63 ++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/src/Data/SQLiteContainer.cpp b/src/Data/SQLiteContainer.cpp index 8f0e2fa..242d4dc 100644 --- a/src/Data/SQLiteContainer.cpp +++ b/src/Data/SQLiteContainer.cpp @@ -85,13 +85,13 @@ void SQLiteContainer::SQLiteIterator::step(){ //SQLiteContainer Functions -SQLiteContainer::SQLiteContainer(const string& ref, const string& table) : tableName(table) { +SQLiteContainer::SQLiteContainer(const string& ref, const string& table, int batchSize) : tableName(table) { //Opens the database if possible. if (sqlite3_open(ref.c_str(), &db) != SQLITE_OK) { std::cerr << "Failed to open database: " << sqlite3_errmsg(db) << std::endl; db = nullptr; return; - } + } //Creates a table if needed. string statement = @@ -106,12 +106,19 @@ SQLiteContainer::SQLiteContainer(const string& ref, const string& table) : table sqlite3_close(db); db = nullptr; } + + BATCH_SIZE = batchSize; } SQLiteContainer::~SQLiteContainer(){ + //Commits everything not yet saved. + if(transactionInProgress){ + commitTransaction(); + } if (db) { sqlite3_close(db); - } + db = nullptr; + } } //Methods @@ -173,6 +180,10 @@ bool SQLiteContainer::empty() const{ } void SQLiteContainer::clear(){ + if (transactionInProgress) { + commitTransaction(); + } + sqlite3_stmt* stmt; string sql = "DELETE FROM " + tableName + ";"; @@ -189,8 +200,11 @@ void SQLiteContainer::clear(){ } bool SQLiteContainer::remove(const shared_ptr& val){ + if (!transactionInProgress) { + beginTransaction(); + } + sqlite3_stmt* stmt; - int sizeBefore = size(); string data = val->to_string(); string statement = "DELETE FROM " + tableName + " WHERE data=? LIMIT 1"; @@ -210,10 +224,18 @@ bool SQLiteContainer::remove(const shared_ptr& val){ sqlite3_finalize(stmt); - return sizeBefore < size(); + if (++operationCount >= BATCH_SIZE) { + commitTransaction(); + } + + return true; } void SQLiteContainer::add(const shared_ptr& val){ + if (!transactionInProgress) { + beginTransaction(); + } + sqlite3_stmt* stmt; string statement = "INSERT INTO " + tableName + " (data) VALUES (?);"; @@ -225,7 +247,6 @@ void SQLiteContainer::add(const shared_ptr& val){ sqlite3_finalize(stmt); throw std::runtime_error("Failed to bind data"); } - // Execute the statement if (sqlite3_step(stmt) != SQLITE_DONE) { string err = sqlite3_errmsg(db); @@ -236,8 +257,36 @@ void SQLiteContainer::add(const shared_ptr& val){ // Get the auto-generated ID int inserted_id = static_cast(sqlite3_last_insert_rowid(db)); val->setObjectID(inserted_id); - // Finalize to free the statement object sqlite3_finalize(stmt); + + if (++operationCount >= BATCH_SIZE) { + commitTransaction(); + } +} + +void SQLiteContainer::beginTransaction() { + char* errMsg = nullptr; + if (sqlite3_exec(db, "BEGIN TRANSACTION;", nullptr, nullptr, &errMsg) != SQLITE_OK) { + std::cerr << "Failed to begin transaction: " << errMsg << std::endl; + sqlite3_free(errMsg); + }else{ + operationCount = 0; + transactionInProgress = true; + } +} + +void SQLiteContainer::commitTransaction() { + char* errMsg = nullptr; + if (sqlite3_exec(db, "COMMIT;", nullptr, nullptr, &errMsg) != SQLITE_OK) { + std::cerr << "Failed to commit transaction: " << errMsg << std::endl; + sqlite3_free(errMsg); + } + operationCount = 0; + transactionInProgress = false; +} + +void SQLiteContainer::setBatchSize(int newSize){ + BATCH_SIZE = newSize; } #endif From 16ad72acff213d674a86bbc79843dc4deb825a88 Mon Sep 17 00:00:00 2001 From: GregoryFan Date: Wed, 20 Aug 2025 18:41:12 +0900 Subject: [PATCH 9/9] Moved commitTransaction and beginTransaction to virtual database functions. Uncommented USE_SQLITE flag in SQLiteContainer. --- include/GenSync/Data/DatabaseContainer.h | 16 +++++++++++++++- include/GenSync/Data/SQLiteContainer.h | 8 ++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/include/GenSync/Data/DatabaseContainer.h b/include/GenSync/Data/DatabaseContainer.h index c7ea5b4..ff3e3e0 100644 --- a/include/GenSync/Data/DatabaseContainer.h +++ b/include/GenSync/Data/DatabaseContainer.h @@ -68,7 +68,21 @@ class DatabaseContainer : public DataContainer { * Pushes a given DataObject into the container for storage. * @param val The given DataObject to store. */ - virtual void add (const shared_ptr& val) override = 0; + virtual void add(const shared_ptr& val) override = 0; + + protected: + + /** + * Begins a buffer for committing actions onto the database. + * Not all databases support transactions. + */ + virtual void beginTransaction(){}; + + /** + * Sends a buffer to be comitted onto the database. + * Not all databases support transactions. + */ + virtual void commitTransaction(){}; }; #endif \ No newline at end of file diff --git a/include/GenSync/Data/SQLiteContainer.h b/include/GenSync/Data/SQLiteContainer.h index 6fc01b6..3e40820 100644 --- a/include/GenSync/Data/SQLiteContainer.h +++ b/include/GenSync/Data/SQLiteContainer.h @@ -2,7 +2,7 @@ // Created by GregoryFan on 7/10/2025 // -//#ifdef USE_SQLITE +#ifdef USE_SQLITE #ifndef SQLITE_CONTAINER_H #define SQLITE_CONTAINER_H @@ -206,12 +206,12 @@ class SQLiteContainer : public DatabaseContainer { /** * Begins a buffer that stores given commands to the disk. */ - void beginTransaction(); + void beginTransaction() override; /** * Executs all commands within the buffer to the disk. */ - void commitTransaction(); + void commitTransaction() override; }; #endif //SQLITE_CONTAINER_H -//#endif \ No newline at end of file +#endif \ No newline at end of file