sync optimizations

cpp/example/test.c

@@ -81,45 +81,52 @@ int main(){
    }
 
    buffer b4 ;
-   b4.len = 153600;
-   b4.data = (unsigned char*)malloc(153600); 
+   b4.len = 0;
+   b4.data = (unsigned char*)malloc(37*sizeof(unsigned char));
 
    printf("storage size of st2 is %d \n",storage_size(st2));
-
-   size_t outSize = negentropy_initiate(ngn_inst1, &b4);
-   if(outSize == 0){
+   result res;
+   int ret1 = negentropy_initiate(ngn_inst1, &res);
+   if(ret1 < 0){
     perror("failed to initiate negentropy instance");
    }
-   printf("initiated negentropy successfully with output of len %zu \n", outSize);
-   b4.len = outSize;
+   printf("initiated negentropy successfully with output of len %llu \n", res.output.len);
+   b4.len = res.output.len;
+   memcpy(b4.data, res.output.data, res.output.len); 
+   free_result(&res);
 
    buffer b3 ;
-   b3.len = 153600;
-   b3.data = (unsigned char*)malloc(153600);
+   b3.len = 0;
+   b3.data = (unsigned char*)malloc(69*sizeof(unsigned char));
 
-   outSize = reconcile(ngn_inst2, &b4, &b3); 
-   if(outSize == 0){
-    perror("nothing to reconcile");
+   ret1 = reconcile(ngn_inst2, &b4, &res); 
+   if(ret1 < 0){
+    perror("error from reconcile");
    }
-   printf("reconcile returned with output of len %zu \n", outSize);
-   b3.len = outSize;
-
+   if (res.output.len == 0){
+      perror("nothing to reconcile");
+   } 
+   printf("reconcile returned with output of len %llu \n", res.output.len);
+   b3.len = res.output.len;
+   memcpy(b3.data, res.output.data, res.output.len); 
+   free_result(&res);
    //outSize = reconcile_with_ids(ngn_inst1, &b3, &rec_callback);
 
-   result res;
-   reconcile_with_ids_no_cbk(ngn_inst1, &b3, &res);
-   printf("needIds count:%llu , haveIds count: %llu \n",res.need_ids_len, res.have_ids_len);
+   result res1;
+   reconcile_with_ids_no_cbk(ngn_inst1, &b3, &res1);
+   printf("needIds count:%llu , haveIds count: %llu \n",res1.need_ids_len, res1.have_ids_len);
 
-   for (int i=0; i < res.need_ids_len ; i++) {
+   for (int i=0; i < res1.need_ids_len ; i++) {
       printf("need ID at %d :", i);
-      printHexBuffer(res.need_ids[i]);
+      printHexBuffer(res1.need_ids[i]);
    }
 
-   for (int j=0; j < res.have_ids_len ; j++) {
+   for (int j=0; j < res1.have_ids_len ; j++) {
       printf("need ID at %d :", j);
-      printHexBuffer(res.have_ids[j]);
+      printHexBuffer(res1.have_ids[j]);
    }
 
    free(b3.data);
    free(b4.data);
+   free_result(&res1);
 }

cpp/negentropy.h

@@ -50,7 +50,6 @@ struct Negentropy {
 
         std::string output;
         output.push_back(PROTOCOL_VERSION);
-        std::cout << "storage size:" << storage.size() << std::endl;
         output += splitRange(0, storage.size(), Bound(MAX_U64));
 
         return output;

cpp/negentropy_wrapper.c

@@ -29,7 +29,7 @@ void* storage_new(const char* db_path, const char* name){
         btreeDbi = negentropy::storage::BTreeMem::setupDB(txn, name);
         txn.commit();
     } */
-    //TODO: Finish constructor 
+
     storage = new negentropy::storage::BTreeMem();
     return storage;
 }
@@ -57,15 +57,15 @@ void* negentropy_new(void* storage, uint64_t frameSizeLimit){
 
     Negentropy<negentropy::storage::BTreeMem>* ne;
     try{
-    ne = new Negentropy<negentropy::storage::BTreeMem>(*lmdbStorage, frameSizeLimit);
+        ne = new Negentropy<negentropy::storage::BTreeMem>(*lmdbStorage, frameSizeLimit);
     }catch(negentropy::err e){
-        //TODO:Find a way to return this error
         return NULL;
     }
     return ne;
 }
 
-size_t negentropy_initiate(void* negentropy, buffer* out){
+// Returns -1 if already initiated.
+int negentropy_initiate(void* negentropy, result* result){
     Negentropy<negentropy::storage::BTreeMem>* ngn_inst;
     ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
 
@@ -75,14 +75,19 @@ size_t negentropy_initiate(void* negentropy, buffer* out){
 /*         std::cout << "output of initiate is, len:" << output->size() << ", output:";
         printHexString(std::string_view(*output)); */
     } catch(negentropy::err e){
-        std::cout << "Exception raised in initiate " << e.what() << std::endl;
-        //TODO:Find a way to return this error
-        return 0;
+        //std::cout << "Exception raised in initiate " << e.what() << std::endl;
+        return -1;
+    }
+    if (output->size() > 0 ){
+        result->output.len = output->size();
+        result->output.data = (unsigned char*)calloc(output->size(), sizeof(unsigned char));
+        memcpy(result->output.data, (unsigned char*)output->c_str(),result->output.len) ;
+    }else {
+        result->output.len = 0;
+        result->output.data = NULL;
     }
-    memcpy( out->data, output->c_str() ,output->size());
-    size_t outlen = output->size();
     delete output;
-    return outlen;
+    return 0;
 }
 
 void negentropy_setinitiator(void* negentropy){
@@ -118,7 +123,7 @@ bool storage_erase(void* storage, uint64_t createdAt, buffer* id){
     return lmdbStorage->erase(createdAt, data);
 }
 
-size_t reconcile(void* negentropy, buffer* query, buffer* output){
+int reconcile(void* negentropy, buffer* query, result* result){
     Negentropy<negentropy::storage::BTreeMem> *ngn_inst;
     ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
     std::string* out = new std::string();
@@ -127,12 +132,22 @@ size_t reconcile(void* negentropy, buffer* query, buffer* output){
 /*         std::cout << "reconcile output of reconcile is, len:" << out->size() << ", output:";
         printHexString(std::string_view(*out)); */
     } catch(negentropy::err e){
-        //TODO:Find a way to return this error
-        std::cout << "Exception raised in reconcile " << e.what() << std::endl;
-        return 0;
+        //All errors returned are non-recoverable errors.
+        //So passing on the error message upwards
+        //std::cout << "Exception raised in reconcile " << e.what() << std::endl;
+        result->error = (char*)calloc(strlen(e.what()), sizeof(char));
+        strcpy(result->error,e.what());
+        return -1;
+    }
+    if (out->size() > 0 ){
+        result->output.len = out->size();
+        result->output.data = (unsigned char*)calloc(out->size(), sizeof(unsigned char));
+        memcpy(result->output.data, (unsigned char*)out->c_str(),result->output.len) ;
+    }else {
+        result->output.len = 0;
+        result->output.data = NULL;
     }
-    memcpy( output->data, out->c_str() ,out->size());
-    return out->size();
+    return 0;
 }
 
 void transform(std::vector<std::string> &from_ids, buffer* to_ids)
@@ -198,7 +213,7 @@ void transform_with_alloc(std::vector<std::string> &from_ids, buffer* to_ids)
    }
 }
 
-void reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result){
+int reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result){
     Negentropy<negentropy::storage::BTreeMem> *ngn_inst;
     ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
 
@@ -223,8 +238,9 @@ void reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result)
 
     } catch(negentropy::err e){
         std::cout << "caught error "<< e.what() << std::endl;
-        //TODO:Find a way to return this error and cleanup partially allocated memory if any
-        return ;
+        result->error = (char*)calloc(strlen(e.what()), sizeof(char));
+        strcpy(result->error,e.what());
+        return -1;
     }
     buffer output = {0,NULL};
     if (out) {
@@ -238,7 +254,7 @@ void reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result)
         result->output.len = 0;
         result->output.data = NULL;
     }
-    return ;
+    return 0;
 }
 
 //Note: This function assumes that all relevant heap memory is alloced and just tries to free
@@ -260,4 +276,8 @@ void free_result(result* r){
         }
         free((void *)r->need_ids);
     }
+
+    if (r->error != NULL && strlen(r->error) > 0){
+        free((void *)r->error);
+    }
 }

cpp/negentropy_wrapper.h

@@ -19,6 +19,7 @@ typedef struct _result_ {
   uint64_t need_ids_len;
   buffer* have_ids;
   buffer* need_ids;
+  char* error;
 } result;
 
 //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code.
@@ -34,21 +35,21 @@ EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit);
 
 EXTERNC void negentropy_delete(void* negentropy);
 
-EXTERNC size_t negentropy_initiate(void* negentropy, buffer* output);
+EXTERNC int negentropy_initiate(void* negentropy, result* result);
 
 EXTERNC void negentropy_setinitiator(void* negentropy);
 
 EXTERNC bool storage_insert(void* storage, uint64_t createdAt, buffer* id);
 
 EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id);
 
-EXTERNC size_t reconcile(void* negentropy, buffer* query, buffer* output);
+EXTERNC int reconcile(void* negentropy, buffer* query, result* result);
 
 EXTERNC typedef void (*reconcile_cbk)(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output, char* outptr );
 
 EXTERNC int reconcile_with_ids(void* negentropy, buffer*  query, reconcile_cbk cbk, char* outptr);
 
-EXTERNC void reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result);
+EXTERNC int reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result);
 
 EXTERNC void free_result(result* result);