Feature/272 manual step api implementation (#275)

Co-authored-by: Maarten de Kruijf <[email protected]>

Author: lucamrgs

docs/content/en/docs/core-components/api-manual.md

@@ -15,8 +15,9 @@ We will use HTTP status codes https://en.wikipedia.org/wiki/List_of_HTTP_status_
 
 ```plantuml
 @startuml
-protocol Reporter {
+protocol Manual {
     GET     /manual
+    GET     /manual/{execution-id}/{step-id}
     POST    /manual/continue
 }
 @enduml
@@ -33,7 +34,7 @@ Get all pending manual actions objects that are currently waiting in SOARCA.
 None
 
 ##### Response
-200/OK with payload list of:
+200/OK with body a list of:
 
 
 
@@ -45,8 +46,8 @@ None
 |step_id            |UUID                   |string             |The id of the step executed by the execution
 |description        |description of the step|string             |The description from the workflow step
 |command            |command                |string             |The command for the agent either command 
-|command_is_base64  |true \| false          |bool               |Indicate the command is in base 64
-|targets            |cacao agent-target     |dictionary         |Map of [cacao agent-target](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509) with the target(s) of this command
+|command_is_base64  |true \| false             |bool               |Indicates if the command is in Base64
+|target            |cacao agent-target     |object         |Map of [cacao agent-target](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509) with the target(s) of this command
 |out_args          |cacao variables        |dictionary         |Map of [cacao variables](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256555) handled in the step out args with current values and definitions
 
 
@@ -97,7 +98,7 @@ Get pending manual actions objects that are currently waiting in SOARCA for spec
 None
 
 ##### Response
-200/OK with payload:
+200/OK with body:
 
 
 
@@ -109,7 +110,7 @@ None
 |step_id            |UUID                   |string             |The id of the step executed by the execution
 |description        |description of the step|string             |The description from the workflow step
 |command            |command                |string             |The command for the agent either command 
-|command_is_base64  |true \| false          |bool               |Indicate the command is in base 64
+|command_is_base64  |true \| false             |bool               |Indicates if the command is in Base64
 |targets            |cacao agent-target     |dictionary         |Map of [cacao agent-target](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509) with the target(s) of this command
 |out_args          |cacao variables        |dictionary         |Map of [cacao variables](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256555) handled in the step out args with current values and definitions
 
@@ -154,7 +155,7 @@ None
 General error
 
 #### POST `/manual/continue`
-Respond to manual command pending in SOARCA, if out_args are defined they must be filled in and returned in the payload body. Only value is required in the response of the variable. You can however return the entire object. Of the object does not match the original out_arg the call we be considered as failed.
+Respond to manual command pending in SOARCA, if out_args are defined they must be filled in and returned in the payload body. Only value is required in the response of the variable. You can however return the entire object. If the object does not match the original out_arg, the call we be considered as failed.
 
 ##### Call payload
 |field              |content                |type               | description |
@@ -163,9 +164,8 @@ Respond to manual command pending in SOARCA, if out_args are defined they must b
 |execution_id       |UUID                   |string             |The id of the execution
 |playbook_id        |UUID                   |string             |The id of the CACAO playbook executed by the execution
 |step_id            |UUID                   |string             |The id of the step executed by the execution
-|response_status    |enum                   |string             |Can be either `success` or `failed`
-|response_out_args  |cacao variables        |dictionary         |Map of [cacao variables](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256555) handled in the step out args with current values and definitions
-
+|response_status    |enum                   |string             |`success` indicates successfull fulfilment of the manual request. `failure` indicates failed satisfaction of the request
+|response_out_args  |cacao variables        |dictionary         |Map of cacao variables names to cacao variable struct. Only name, type, and value are mandatory
 
 
 ```plantuml
@@ -176,15 +176,15 @@ Respond to manual command pending in SOARCA, if out_args are defined they must b
         "execution_id" : "<execution-id>",
         "playbook_id" :  "<playbook-id>",
         "step_id" :  "<step-id>",
-        "response_status" : "success | failed",
+        "response_status" : "success | failure",
         "response_out_args":    {
             "<variable-name-1>" : {
-                "type":         "<type>",
+                "type":         "<variable-type>",
                 "name":         "<variable-name>",
-                "description":  "<description>",
                 "value":        "<value>",
-                "constant":     "<true/false>",
-                "external":     "<true/false>"
+                "description":  "<description> (ignored)",
+                "constant":     "<true/false> (ignored)",
+                "external":     "<true/false> (ignored)"
             }
         }
     }
@@ -193,7 +193,8 @@ Respond to manual command pending in SOARCA, if out_args are defined they must b
 ```
 
 ##### Response
-200/OK with payload:
+200/OK with payload: 
+Generic execution information
 
 ##### Error
 400/BAD REQUEST with payload:

docs/content/en/docs/core-components/modules.md

@@ -322,20 +322,199 @@ This example will start an operation that executes the ability with ID `36eecb80
 ```
 
 ### Manual capability 
-This capability executes [manual Commands](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256491) and provides them through the [SOARCA api](/docs/core-components/api-manual).
+This capability executes [manual Commands](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256491) and provides them natively through the [SOARCA api](/docs/core-components/api-manual), though other integrations are possible.
 
 
-<!-- The manual capability will allow an operator to interact with a playbook. It could allow one to perform a manual step that could not be automated, enter a variable to the playbook execution or a combination of these operations.
+The manual capability will allow an operator to interact with a playbook. It could allow one to perform a manual step that could not be automated, enter a variable to the playbook execution or a combination of these operations.
 
-The main way to interact with the manual step is through SOARCA's [manual api](/docs/core-components/api-manual). The manual step should provide a timeout SOARCA will by default use a timeout of 10 minutes. If a timeout occurs the step is considered as failed. -->
+The manual step should provide a timeout. SOARCA will by default use a timeout of 10 minutes. If a timeout occurs, the step is considered as failed.
 
+#### Manual capability architecture
 
+In essence, executing a manual command involves the following actions:
+1. A message, the `command` of a manual command, is posted *somewhere*, *somehow*, together with the variables of which values is expected to be assigned or updated (if any).
+2. The playbook execution stops, waiting for *something* to respond to the message with the variables values.
+3. Once something replies, the variables are streamed inside the playbook execution and handled accordingly.
 
+It should be possible to post a manual command message anywhere and in any way, and allow anything to respond back. Hence, SOARCA adopts a flexible architecture to accomodate different ways of manual *interactions*. Below a view of the architecture.
 
+When a playbook execution hits an Action step with a manual command, the *ManualCapability* will queue the instruction into the *CapabilityInteraction* module. The module does essentially three things: 
+1. it stores the status of the manual command, and implements the SOARCA API interactions with the manual command.
+2. If manual integrations are defined for the SOARCA instance, the *CapabilityInteraction* module notifies the manual integration modules, so that they can handle the manual command in turn.
+3. It waits for the manual command to be satisfied either via SOARCA APIs, or via manual integrations. The first to respond amongst the two, resolves the manual command. The resolution of the command may or may not assign new values to variables in the playbook. Finally the *CapabilityInteraction* module replies to the *ManualCommand* module.
+
+Ultimately the *ManualCapability* then completes its execution, having eventually updated the values for the variables in the outArgs of the command. Timeouts or errors are handled opportunely.
+
+```plantuml
+@startuml
+set separator ::
+
+class ManualCommand 
+
+protocol ManualAPI {
+    GET     /manual
+    GET     /manual/{exec-id}/{step-id}
+    POST    /manual/continue
+}
+
+interface ICapability{
+   Execute()
+}
+
+interface ICapabilityInteraction{
+    Queue(command InteractionCommand, manualComms ManualCapabilityCommunication)
+}
+
+interface IInteracionStorage{
+    GetPendingCommands() []CommandData
+	GetPendingCommand(execution.metadata) CommandData
+	PostContinue(execution.metadata) ExecutionInfo
+}
+
+interface IInteractionIntegrationNotifier {
+    Notify(command InteractionIntegrationCommand, channel manualCapabilityCommunication.Channel)
+}
+
+class Interaction {
+    notifiers []IInteractionIntegrationNotifier
+    storage map[executionId]map[stepId]InteractionStorageEntry
+}
+class ThirdPartyManualIntegration
+
+
+ManualCommand .up.|> ICapability
+ManualCommand -down-> ICapabilityInteraction
+Interaction .up.|> ICapabilityInteraction
+Interaction .up.|> IInteracionStorage
+
+ManualAPI -down-> IInteracionStorage
+
+Interaction -right-> IInteractionIntegrationNotifier
+ThirdPartyManualIntegration .up.|> IInteractionIntegrationNotifier
+
+
+```
+
+The default and internally-supported way to interact with the manual step is through SOARCA's [manual api](/docs/core-components/api-manual). 
+Besides SOARCA's [manual api](/docs/core-components/api-manual), SOARCA is designed to allow for configuration of additional ways that a manual command should be executed. In particular, there can be *one* manual integration (besides the native manual APIs) per running SOARCA instance.
+Integration's code should implement the *IInteractionIntegrationNotifier* interface, returning the result of the manual command execution in form of an `InteractionIntegrationResponse` object, into the respective channel.
+
+The diagram below displays in some detail the way the manual interactions components work.
+
+```plantuml
+@startuml
+control "ManualCommand" as manual
+control "Interaction" as interaction
+control "ManualAPI" as api
+control "ThirdPartyManualIntegration" as 3ptool
+participant "Integration" as integration
+
+-> manual : ...manual command 
+manual -> interaction : Queue(command, capabilityChannel, timeoutContext)
+manual -> manual : idle wait on capabilityChannel
+activate manual
+
+activate interaction
+interaction -> interaction : save pending manual command
+interaction ->> 3ptool : Notify(command, capabilityChannel, timeoutContext)
+3ptool <--> integration : custom handling command posting
+deactivate interaction
+
+alt Command Response
+
+    group Native ManualAPI flow
+        api -> interaction : GetPendingCommands()
+        activate interaction
+        activate api
+        api -> interaction : GetPendingCommand(execution.metadata)
+        api -> interaction : PostContinue(ManualOutArgsUpdate)
+        interaction -> interaction : build InteractionResponse
+        deactivate api
+        interaction --> manual : capabilityChannel <- InteractionResponse
+        manual ->> interaction : timeoutContext.Cancel() event
+        interaction -> interaction : de-register pending command
+        deactivate interaction
+        manual ->> 3ptool : timeoutContext.Deadline() event
+        activate 3ptool
+        3ptool <--> integration : custom handling command completed
+        deactivate manual
+        <- manual : ...continue execution
+        deactivate 3ptool
+        deactivate integration
+    end
+else 
+    group Third Party Integration flow
+        integration --> 3ptool : custom handling command response
+        activate manual
+        activate integration
+        deactivate integration
+        activate 3ptool
+        3ptool -> 3ptool : build InteractionResponse
+        3ptool --> manual : capabilityChannel <- InteractionResponse
+        deactivate 3ptool
+        manual ->> interaction : timeoutContext.Cancel() event
+        activate interaction
+        interaction -> interaction : de-register pending command
+        deactivate interaction
+        manual ->> 3ptool : timeoutContext.Deadline() event
+        activate 3ptool
+        3ptool <--> integration : custom handling command completed
+        deactivate 3ptool
+        activate integration
+        deactivate integration
+        <- manual : ...continue execution
+        deactivate manual
+        deactivate integration
+    end
+end
+
+@enduml
+```
+
+Note that whoever resolves the manual command first, whether via the manualAPI, or a third party integration, then the command results are returned to the workflow execution, and the manual command is removed from the pending list. Hence, if a manual command is resolved e.g. via the manual integration, a postContinue API call for that same command will not go through, as the command will have been resolved already, and hence removed from the registry of pending manual commands.
+
+The diagram below shows instead what happens when a timeout occurs for the manual command.
+
+```plantuml
+@startuml
+control "ManualCommand" as manual
+control "Interaction" as interaction
+control "ManualAPI" as api
+control "ThirdPartyManualIntegration" as 3ptool
+participant "Integration" as integration
+
+-> manual : ...manual command 
+manual -> interaction : Queue(command, capabilityChannel, timeoutContext)
+manual -> manual : idle wait on capabilityChannel
+activate manual
+
+activate interaction
+interaction -> interaction : save pending manual command
+interaction ->> 3ptool : Notify(command, capabilityChannel, timeoutContext)
+3ptool --> integration : custom handling command posting
+deactivate interaction
+
+group Command execution times out
+    manual -> manual : timeoutContext.Deadline()
+    manual ->> interaction : timeoutContext.Deadline() event
+    manual ->> 3ptool : timeoutContext.Deadline() event
+    3ptool --> integration : custom handling command timed-out view
+    activate interaction
+    interaction -> interaction : de-register pending command
+    <- manual : ...continue execution
+    deactivate manual
+    ...
+    api -> interaction : GetPendingCommand(execution.metadata)
+    interaction -> api : no pending command (404) 
+end
+
+
+@enduml
+```
 
 #### Success and failure
 
-In SOARCA the manual step is considered successful if a response is made through the [manual api](/docs/core-components/api-manual). The manual command can specify a timeout but if none is specified SOARCA will use a default timeout of 10 minutes. If a timeout occurs the step is considered as failed and SOARCA will return an error to the decomposer.
+In SOARCA the manual step is considered successful if a response is made through the [manual api](/docs/core-components/api-manual), or an integration. The manual command can specify a timeout, but if none is specified SOARCA will use a default timeout of 10 minutes. If a timeout occurs the step is considered as failed and SOARCA will return an error to the decomposer.
 
 #### Variables
 

examples/manual-playbook.json

@@ -0,0 +1,89 @@
+{
+    "type": "playbook",
+    "spec_version": "cacao-2.0",
+    "id": "playbook--fe65ef7b-e8b1-4ed9-ba60-3c380ae5ab28",
+    "name": "Example manual",
+    "description": "This playbook is to demonstrate the manual command definition",
+    "playbook_types": [
+        "notification"
+    ],
+    "created_by": "identity--ac3c0258-7a81-46e7-a2ae-d34b6d06cc54",
+    "created": "2025-01-21T14:14:23.263Z",
+    "modified": "2025-01-21T14:14:23.263Z",
+    "revoked": false,
+    "valid_from": "2023-11-20T15:56:00.123Z",
+    "valid_until": "2123-11-20T15:56:00.123Z",
+    "priority": 1,
+    "severity": 1,
+    "impact": 1,
+    "labels": [
+        "soarca",
+        "manual"
+    ],
+    "external_references": [
+        {
+            "description": "TNO COSSAS"
+        }
+    ],
+    "workflow_start": "start--9e7d62b2-88ac-4656-94e1-dbd4413ba008",
+    "workflow_exception": "end--a6f0b81e-affb-4bca-b4f6-a2d5af908958",
+    "workflow": {
+        "start--9e7d62b2-88ac-4656-94e1-dbd4413ba008": {
+            "name": "Start example flow for manual command",
+            "on_completion": "action--eb9372d4-d524-49fc-bf24-be26ea084779",
+            "type": "start"
+        },
+        "action--eb9372d4-d524-49fc-bf24-be26ea084779": {
+            "name": "manual",
+            "description": "Instruction to the operator to be executed manually",
+            "step_variables": {
+                "__hyperspeed_ready__": {
+                    "type": "string",
+                    "description": "set value to true or false when the request is completed",
+                    "constant": false,
+                    "external": false
+                }
+            },
+            "on_completion": "end--a6f0b81e-affb-4bca-b4f6-a2d5af908958",
+            "type": "action",
+            "commands": [
+                {
+                    "type": "manual",
+                    "command": "prepare Falcon for hyperspeed jump"
+                }
+            ],
+            "agent": "soarca-manual-capability--7b0e98db-fa93-42aa-8511-e871c65131b1",
+            "targets": [
+                "individual--9d1f6217-34d5-435c-b29a-6a1af6b664d9"
+            ],
+            "out_args": [
+                "__hyperspeed_ready__"
+            ]
+        },
+        "end--a6f0b81e-affb-4bca-b4f6-a2d5af908958": {
+            "name": "End Flow",
+            "type": "end"
+        }
+    },
+    "agent_definitions": {
+        "soarca--00040001-1000-1000-a000-000100010001": {
+            "type": "soarca",
+            "name": "soarca-manual-capability"
+        },
+        "soarca-manual-capability--7b0e98db-fa93-42aa-8511-e871c65131b1": {
+            "type": "soarca-manual-capability",
+            "name": "soarca-manual-capability",
+            "description": "SOARCA's manual command handler"
+        }
+    },
+    "target_definitions": {
+        "individual--9d1f6217-34d5-435c-b29a-6a1af6b664d9": {
+            "type": "individual",
+            "name": "Luke Skywalker",
+            "description": "Darth Vader's son",
+            "location": {
+                "name": "Somewhere in a galaxy far far away"
+            }
+        }
+    }
+}