Implemented out-of-proc error reporting schema (#196)

Author: davidmrdavid

azure/durable_functions/orchestrator.py

@@ -100,13 +100,22 @@ def handle(self, context: DurableOrchestrationContext):
                     actions=self.durable_context.actions,
                     custom_status=self.durable_context.custom_status)
             except Exception as e:
+                exception_str = str(e)
                 orchestration_state = OrchestratorState(
                     is_done=False,
                     output=None,  # Should have no output, after generation range
                     actions=self.durable_context.actions,
-                    error=str(e),
+                    error=exception_str,
                     custom_status=self.durable_context.custom_status)
 
+                # Create formatted error, using out-of-proc error schema
+                error_label = "\n\n$OutOfProcData$:"
+                state_str = orchestration_state.to_json_string()
+                formatted_error = f"{exception_str}{error_label}{state_str}"
+
+                # Raise exception, re-set stack to original location
+                raise Exception(formatted_error) from e
+
         # No output if continue_as_new was called
         if self.durable_context.will_continue_as_new:
             orchestration_state._output = None

tests/orchestrator/test_call_http.py

@@ -104,17 +104,25 @@ def test_failed_state():
     add_failed_http_events(
         context_builder, 0, failed_reason, failed_details)
 
-    result = get_orchestration_state_result(
-        context_builder, simple_get_generator_function)
-
-    expected_state = base_expected_state()
-    request = get_request()
-    add_http_action(expected_state, request)
-    expected_state._error = f'{failed_reason} \n {failed_details}'
-    expected = expected_state.to_json()
-
-    assert_valid_schema(result)
-    assert_orchestration_state_equals(expected, result)
+    try:
+        result = get_orchestration_state_result(
+            context_builder, simple_get_generator_function)
+        # We expected an exception
+        assert False
+    except Exception as e:
+        error_label = "\n\n$OutOfProcData$:"
+        error_str = str(e)
+
+        expected_state = base_expected_state()
+        request = get_request()
+        add_http_action(expected_state, request)
+
+        error_msg = f'{failed_reason} \n {failed_details}'
+        expected_state._error = error_msg
+        state_str = expected_state.to_json_string()
+        
+        expected_error_str = f"{error_msg}{error_label}{state_str}"
+        assert expected_error_str == error_str
 
 
 def test_initial_post_state():

tests/orchestrator/test_fan_out_fan_in.py

@@ -153,13 +153,22 @@ def test_failed_parrot_value():
     add_completed_task_set_events(context_builder, 1, 'ParrotValue', activity_count,
                                   2, failed_reason, failed_details)
 
-    result = get_orchestration_state_result(
-        context_builder, generator_function)
-
-    expected_state = base_expected_state(error=f'{failed_reason} \n {failed_details}')
-    add_single_action(expected_state, function_name='GetActivityCount', input_=None)
-    add_multi_actions(expected_state, function_name='ParrotValue', volume=activity_count)
-    expected = expected_state.to_json()
-
-    assert_valid_schema(result)
-    assert_orchestration_state_equals(expected, result)
+    try:
+        result = get_orchestration_state_result(
+            context_builder, generator_function)
+        # we expected an exception
+        assert False
+    except Exception as e:
+        error_label = "\n\n$OutOfProcData$:"
+        error_str = str(e)
+
+        expected_state = base_expected_state(error=f'{failed_reason} \n {failed_details}')
+        add_single_action(expected_state, function_name='GetActivityCount', input_=None)
+        add_multi_actions(expected_state, function_name='ParrotValue', volume=activity_count)
+
+        error_msg = f'{failed_reason} \n {failed_details}'
+        expected_state._error = error_msg
+        state_str = expected_state.to_json_string()
+        
+        expected_error_str = f"{error_msg}{error_label}{state_str}"
+        assert expected_error_str == error_str

tests/orchestrator/test_retries.py

@@ -255,9 +255,16 @@ def test_retries_can_fail():
     """Tests the code path where a retry'ed Task fails"""
     context = get_context_with_retries(will_fail=True)
 
-    result = get_orchestration_state_result(
-        context, generator_function)
-
-    expected_error = f"{REASONS} \n {DETAILS}"
-    assert "error" in result
-    assert result["error"] == expected_error
+    try:
+        result = get_orchestration_state_result(
+            context, generator_function)
+        # We expected an exception
+        assert False
+    except Exception as e:
+        error_label = "\n\n$OutOfProcData$:"
+        error_str = str(e)
+
+        error_msg = f"{REASONS} \n {DETAILS}"
+        
+        expected_error_str = f"{error_msg}{error_label}"
+        assert str.startswith(error_str, expected_error_str)

tests/orchestrator/test_sequential_orchestrator.py

@@ -20,6 +20,18 @@ def generator_function(context):
 
     return outputs
 
+def generator_function_rasing_ex(context):
+    outputs = []
+
+    task1 = yield context.call_activity("Hello", "Tokyo")
+    task2 = yield context.call_activity("Hello", "Seattle")
+    task3 = yield context.call_activity("Hello", "London")
+
+    outputs.append(task1)
+    outputs.append(task2)
+    outputs.append(task3)
+
+    raise ValueError("Oops!")
 
 def generator_function_with_serialization(context):
     """Ochestrator to test sequential activity calls with a serializable input arguments."""
@@ -99,17 +111,50 @@ def test_failed_tokyo_state():
     add_hello_failed_events(
         context_builder, 0, failed_reason, failed_details)
 
-    result = get_orchestration_state_result(
-        context_builder, generator_function)
-
-    expected_state = base_expected_state()
-    add_hello_action(expected_state, 'Tokyo')
-    expected_state._error = f'{failed_reason} \n {failed_details}'
-    expected = expected_state.to_json()
-
-    assert_valid_schema(result)
-    assert_orchestration_state_equals(expected, result)
+    try:
+        result = get_orchestration_state_result(
+            context_builder, generator_function)
+        # expected an exception
+        assert False
+    except Exception as e:
+        error_label = "\n\n$OutOfProcData$:"
+        error_str = str(e)
+
+        expected_state = base_expected_state()
+        add_hello_action(expected_state, 'Tokyo')
+        error_msg = f'{failed_reason} \n {failed_details}'
+        expected_state._error = error_msg
+        state_str = expected_state.to_json_string()
+        
+        expected_error_str = f"{error_msg}{error_label}{state_str}"
+        assert expected_error_str == error_str
+
+
+def test_user_code_raises_exception():
+    context_builder = ContextBuilder('test_simple_function')
+    add_hello_completed_events(context_builder, 0, "\"Hello Tokyo!\"")
+    add_hello_completed_events(context_builder, 1, "\"Hello Seattle!\"")
+    add_hello_completed_events(context_builder, 2, "\"Hello London!\"")
 
+    try:
+        result = get_orchestration_state_result(
+            context_builder, generator_function_rasing_ex)
+        # expected an exception
+        assert False
+    except Exception as e:
+        error_label = "\n\n$OutOfProcData$:"
+        error_str = str(e)
+
+        expected_state = base_expected_state()
+        add_hello_action(expected_state, 'Tokyo')
+        add_hello_action(expected_state, 'Seattle')
+        add_hello_action(expected_state, 'London')
+        error_msg = 'Oops!'
+        expected_state._error = error_msg
+        state_str = expected_state.to_json_string()
+        
+        expected_error_str = f"{error_msg}{error_label}{state_str}"
+        assert expected_error_str == error_str
 
 def test_tokyo_and_seattle_state():
     context_builder = ContextBuilder('test_simple_function')

tests/orchestrator/test_sequential_orchestrator_with_retry.py

@@ -198,13 +198,21 @@ def test_failed_tokyo_hit_max_attempts():
     add_hello_failed_events(context_builder, 4, failed_reason, failed_details)
     add_retry_timer_events(context_builder, 5)
 
-    result = get_orchestration_state_result(
-        context_builder, generator_function)
-
-    expected_state = base_expected_state()
-    add_hello_action(expected_state, 'Tokyo')
-    expected_state._error = f'{failed_reason} \n {failed_details}'
-    expected = expected_state.to_json()
-
-    assert_valid_schema(result)
-    assert_orchestration_state_equals(expected, result)
+    try:
+        result = get_orchestration_state_result(
+            context_builder, generator_function)
+        # expected an exception
+        assert False
+    except Exception as e:
+        error_label = "\n\n$OutOfProcData$:"
+        error_str = str(e)
+
+        expected_state = base_expected_state()
+        add_hello_action(expected_state, 'Tokyo')
+
+        error_msg = f'{failed_reason} \n {failed_details}'
+        expected_state._error = error_msg
+        state_str = expected_state.to_json_string()
+        
+        expected_error_str = f"{error_msg}{error_label}{state_str}"
+        assert expected_error_str == error_str

tests/orchestrator/test_sub_orchestrator_with_retry.py

@@ -109,15 +109,21 @@ def test_tokyo_and_seattle_and_london_state_all_failed():
     add_hello_suborch_failed_events(context_builder, 4, failed_reason, failed_details)
     add_retry_timer_events(context_builder, 5)
 
-
-    result = get_orchestration_state_result(
-        context_builder, generator_function)
-
-    expected_state = base_expected_state()
-    add_hello_suborch_action(expected_state, 'Tokyo')
-    expected_state._error = f'{failed_reason} \n {failed_details}'
-    expected = expected_state.to_json()
-    expected_state._is_done = True
-
-    #assert_valid_schema(result)
-    assert_orchestration_state_equals(expected, result)
+    try:
+        result = get_orchestration_state_result(
+            context_builder, generator_function)
+        # Should have error'ed out
+        assert False
+    except Exception as e:
+        error_label = "\n\n$OutOfProcData$:"
+        error_str = str(e)
+    
+        expected_state = base_expected_state()
+        add_hello_suborch_action(expected_state, 'Tokyo')
+
+        error_msg = f'{failed_reason} \n {failed_details}'
+        expected_state._error = error_msg
+        state_str = expected_state.to_json_string()
+        
+        expected_error_str = f"{error_msg}{error_label}{state_str}"
+        assert expected_error_str == error_str