Merge pull request #71 from SubstraFoundation/fix-mvcc-retry

Fix MVCC Read Conflict retry

Author: samlesu

backend/backend/settings/dev.py

@@ -13,7 +13,7 @@
     'CACHE_DOCKER_IMAGES': to_bool(os.environ.get('TASK_CACHE_DOCKER_IMAGES', False)),
 }
 
-LEDGER_CALL_RETRY = False  # Overwrite the ledger setting value
+# LEDGER_CALL_RETRY = False  # uncomment to overwrite the ledger setting value
 
 # Database
 # https://docs.djangoproject.com/en/2.0/ref/settings/#databases

backend/backend/settings/test.py

@@ -4,3 +4,5 @@
 
 import logging
 logging.disable(logging.CRITICAL)
+
+LEDGER_CALL_RETRY = False

backend/substrapp/ledger_utils.py

@@ -86,7 +86,7 @@ def retry_on_error(delay=1, nbtries=5, backoff=2):
     def _retry(fn):
         @functools.wraps(fn)
         def _wrapper(*args, **kwargs):
-            if not getattr(settings, 'LEDGER_CALL_RETRY', False):
+            if not getattr(settings, 'LEDGER_CALL_RETRY', True):
                 return fn(*args, **kwargs)
 
             _delay = delay
@@ -162,6 +162,11 @@ def call_ledger(call_type, fcn, args=None, kwargs=None):
             if hasattr(e, 'details') and 'access denied' in e.details():
                 raise LedgerForbidden(f'Access denied for {(fcn, args)}')
 
+            for arg in e.args:
+                if 'MVCC_READ_CONFLICT' in arg:
+                    logger.error(f'MVCC read conflict for {(fcn, args)}')
+                    raise LedgerMVCCError(arg) from e
+
             try:  # get first failed response from list of protobuf ProposalResponse
                 response = [r for r in e.args[0] if r.response.status != 200][0].response.message
             except Exception:
@@ -171,9 +176,7 @@ def call_ledger(call_type, fcn, args=None, kwargs=None):
         try:
             response = json.loads(response)
         except json.decoder.JSONDecodeError:
-            if response == 'MVCC_READ_CONFLICT':
-                raise LedgerMVCCError(response)
-            elif 'cannot change status' in response:
+            if 'cannot change status' in response:
                 raise LedgerStatusError(response)
             else:
                 raise LedgerBadResponse(response)