Audit Logs Endpoints Scripts Aligments for Xsoar-8 (demisto#29781)

* test

* fix core api

* ExportAuditLogsToFile - add support for xsoar-8

* add ExportAuditLogsToFile UTs

* add forward audit logs uts

* update ut

* validation fixes

* mypy

* bump rns

* update docker

* update docker image

* fix ut

* format

* Bump pack from version CommonScripts to 1.12.25.

* Bump pack from version CommonScripts to 1.12.26.

* cr

* cr fixes

* update

* fix uts

---------

Co-authored-by: Content Bot <[email protected]>

Author: 2 people

Packs/CommonScripts/ReleaseNotes/1_12_26.md

@@ -0,0 +1,7 @@
+
+#### Scripts
+
+##### ExportAuditLogsToFile
+
+- Added support to XSOAR 8.
+- Updated the Docker image to: *demisto/python3:3.10.13.74666*.

Packs/CommonScripts/Scripts/ExportAuditLogsToFile/ExportAuditLogsToFile.py

@@ -6,43 +6,100 @@
 from datetime import date, timedelta
 
 
+def get_audit_logs(res: Dict):
+
+    def get_xsoar6_audit_logs():
+        return res.get('audits') or []
+
+    def get_xsoar8_audit_logs():
+        return (res.get('reply') or {}).get('data') or []
+
+    return get_xsoar6_audit_logs() or get_xsoar8_audit_logs()
+
+
+def get_audit_logs_count(res: Dict):
+    def get_xsoar6_audit_logs_count():
+        return res.get('total') or 0
+
+    def get_xsoar8_audit_logs_count():
+        return res.get('total_count') or 0
+
+    return arg_to_number(get_xsoar6_audit_logs_count() or get_xsoar8_audit_logs_count())
+
+
 def main():   # pragma: no cover
     # get the file type
     file_type = demisto.args().get("output")
 
     # set time back for fetching
     fetch_back_date = date.today() - timedelta(days=int(demisto.args().get("days_back")))
     fetch_from = fetch_back_date.strftime("%Y-%m-%dT00:00:00Z")
-    file_date = fetch_back_date.strftime("%Y-%m-%d")
 
-    # body of the request
-    body = {
-        'fromDate': fetch_from,
-        'query': "",
-        'page': 0,
-        'size': 100
-    }
+    demisto_version: str = get_demisto_version().get("version")
+    demisto.debug(f'The version of XSOAR is: {demisto_version}')
+    if not demisto_version:
+        raise ValueError('Could not get the version of XSOAR')
+
+    page_num = 0
+    size = 100
+
+    if demisto_version.startswith("6"):  # xsoar 6
+        uri = "/settings/audits"
+        body = {
+            'fromDate': fetch_from,
+            'query': "",
+            'page': page_num,
+            'size': size
+        }
+    else:  # xsoar 8
+        uri = "/public_api/v1/audits/management_logs"
+        body = {
+            "request_data": {
+                "search_from": page_num,
+                "search_to": size,
+                "filters": [
+                    {
+                        'field': 'timestamp',
+                        'operator': 'gte',
+                        'value': date_to_timestamp(fetch_back_date)
+                    },
+                ]
+            }
+        }
 
-    # get the logs
-    res = demisto.executeCommand("demisto-api-post", {"uri": "/settings/audits", "body": body})[0]["Contents"][
-        "response"]
+    args = {"uri": uri, "body": body}
+    res = demisto.executeCommand("demisto-api-post", args)
+    demisto.debug(f'demisto-api-post with {args} returned {res}')
+    if is_error(res):
+        raise DemistoException(f'error occurred when trying to retrieve the audit logs using {args=}, error: {res}')
+
+    response = res[0]["Contents"]["response"]
 
     # set the initial counts
-    total = res.get('total', 0)
-    audits = res.get('audits', [])
-    count = 1
+    total = get_audit_logs_count(response)
+    audits = get_audit_logs(response)
+    page_num += 1
 
     # if there are more events than the default size, page through and get them all
     while len(audits) < total:
-        body['page'] = count
-        res = demisto.executeCommand("demisto-api-post", {"uri": "/settings/audits", "body": body})[0]["Contents"][
-            "response"]
-        audits.extend(res.get('audits', []))
-        count += 1
+        if body.get("page"):  # pagination for xsoar-6
+            body["page"] = page_num
+        else:  # pagination for xsoar-8
+            body["request_data"]["search_from"] = page_num  # type: ignore[index]
+        args = {"uri": uri, "body": body}
+        res = demisto.executeCommand("demisto-api-post", args)
+        demisto.debug(f'demisto-api-post with {args} returned {res}')
+        if is_error(res):
+            raise DemistoException(f'error occurred when trying to retrieve the audit logs using {args=}, error: {res}')
+        response = res[0]["Contents"]["response"]
+        audits.extend(get_audit_logs(response))
+        page_num += 1
         # break if this goes crazy, if there are more than 100 pages of audit log entries.
-        if count == 100:
+        if page_num == 100:
             break
 
+    file_date = fetch_back_date.strftime("%Y-%m-%d")
+
     if file_type == "csv":
         # write the results to a CSV
         si = io.StringIO()

Packs/CommonScripts/Scripts/ExportAuditLogsToFile/ExportAuditLogsToFile.yml

@@ -10,7 +10,7 @@ args:
   required: true
 - auto: PREDEFINED
   defaultValue: csv
-  description: Type of File to return, either JSON, or CSV
+  description: Type of File to return, either JSON, or CSV.
   name: output
   predefined:
   - csv
@@ -25,7 +25,7 @@ contentitemexportablefields:
 dependson:
   must:
   - demisto-api-post
-dockerimage: demisto/python3:3.10.12.63474
+dockerimage: demisto/python3:3.10.13.74666
 enabled: true
 name: ExportAuditLogsToFile
 runas: DBotWeakRole

Packs/CommonScripts/Scripts/ExportAuditLogsToFile/ExportAuditLogsToFile_test.py

@@ -1,12 +1,59 @@
 import demistomock as demisto
-from ExportAuditLogsToFile import main
+import pytest
+import ExportAuditLogsToFile
 from freezegun import freeze_time
+from CommonServerPython import *
+
+
+def execute_command_side_effect(command: str, args: Dict):
+    if command == "splunk-submit-event-hec":
+        return [{'Contents': {'response': ["result1"]}}]
+    if command == "demisto-api-post":
+        if args["uri"] == '/settings/audits':
+            return [{'Contents': {'response': {'total': 2, "audits": ["audit1", "audit2"]}}, "Type": entryTypes["note"]}]
+        return [{'Contents': {'response': {'total': 2, "reply": {"data": ["audit1", "audit2"]}}}, "Type": entryTypes["note"]}]
+    return None
 
 
 @freeze_time('2020-04-20')
-def test_main(mocker):
+def test_main_no_logs_xsoar6(mocker):
     mocker.patch.object(demisto, 'args', return_value={'output': 'html', 'days_back': '5'})
-    execute_command_mock = mocker.patch.object(demisto, 'executeCommand',
-                                               return_value=[{'Contents': {'response': {'total': 0}}}])
-    main()
+    execute_command_mock = mocker.patch.object(
+        demisto, 'executeCommand', return_value=[{'Contents': {'response': {'total': 0}}, "Type": entryTypes["note"]}]
+    )
+    mocker.patch.object(ExportAuditLogsToFile, "get_demisto_version", return_value={"version": "6.10"})
+    ExportAuditLogsToFile.main()
     assert execute_command_mock.call_args.args[1]['body']['fromDate'] == '2020-04-15T00:00:00Z'
+
+
+@freeze_time('2020-04-20')
+@pytest.mark.parametrize(
+    'xsoar_version, expected_uri',
+    [
+        ({"version": "6.10"}, "/settings/audits"),
+        ({"version": "8.1"}, '/public_api/v1/audits/management_logs')
+    ]
+)
+def test_main_with_logs(mocker, xsoar_version, expected_uri):
+    """
+    Given:
+        - xsoar version
+    When:
+        - Calling main flow
+    Then:
+        - make sure the correct uri is called for each xsoar version
+        - make sure audit logs are fetched when its xsoar 6/8
+    """
+    mocker.patch.object(demisto, 'args', return_value={'output': 'json', 'days_back': '5'})
+    execute_command_mocker = mocker.patch.object(
+        demisto,
+        'executeCommand',
+        side_effect=execute_command_side_effect
+    )
+    mocker.patch.object(ExportAuditLogsToFile, "fileResult", return_value=["audit1", "audit2"])
+    mocker.patch.object(ExportAuditLogsToFile, "get_demisto_version", return_value=xsoar_version)
+    demisto_results_mocker = mocker.patch.object(demisto, "results", return_value=[])
+    ExportAuditLogsToFile.main()
+    assert demisto_results_mocker.called
+    assert "Fetched 2 audit log events" in demisto_results_mocker.call_args_list[1][0][0]
+    assert execute_command_mocker.call_args_list[0][0][1]["uri"] == expected_uri

Packs/CommonScripts/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Common Scripts",
     "description": "Frequently used scripts pack.",
     "support": "xsoar",
-    "currentVersion": "1.12.25",
+    "currentVersion": "1.12.26",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/DemistoRESTAPI/Integrations/CoreRESTAPI/CoreRESTAPI.js

@@ -1,5 +1,13 @@
 
 const MIN_HOSTED_XSOAR_VERSION = '8.0.0';
+// list was taken from https://docs-cortex.paloaltonetworks.com/r/Cortex-XSOAR-8-API
+const SYSTEM_ENDPOINTS = [
+  "/public_api/v1/audits/management_logs",
+  "/public_api/v1/rbac/get_roles",
+  "/public_api/v1/rbac/get_user_group",
+  "/public_api/v1/rbac/get_users",
+  "/public_api/v1/rbac/set_user_role"
+]
 
 var serverURL = params.url;
 if (serverURL.slice(-1) === '/') {
@@ -17,8 +25,9 @@ isHosted = function () {
 }
 
 // only when using XSIAM or XSOAR >= 8.0 we will add the /xsoar suffix
+// and only when it is not a system endpoint (note: this list does not include all system endpoints!)
 if  (isHosted()) {
-    if (!serverURL.endsWith('/xsoar')){
+    if ((!serverURL.endsWith('/xsoar')) && (!SYSTEM_ENDPOINTS.includes(args.uri))) {
         serverURL = serverURL + '/xsoar'
     }
 }

Packs/DemistoRESTAPI/ReleaseNotes/1_3_37.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Core REST API
+
+- Fixed an issue where the integration would not work with XSOAR endpoints that do not start with the **/xsoar** prefix.

Packs/DemistoRESTAPI/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Cortex REST API",
     "description": "Use Demisto REST APIs",
     "support": "xsoar",
-    "currentVersion": "1.3.36",
+    "currentVersion": "1.3.37",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/ForwardXSOARAuditLogsToSplunkHEC/ReleaseNotes/1_0_1.md

@@ -0,0 +1,7 @@
+
+#### Scripts
+
+##### ForwardAuditLogsToSplunkHEC
+
+- Added support for XSOAR 8.
+- Updated the Docker image to: *demisto/python3:3.10.13.74666*.

Packs/ForwardXSOARAuditLogsToSplunkHEC/Scripts/ForwardAuditLogsToSplunkHEC/ForwardAuditLogsToSplunkHEC.py

@@ -4,12 +4,36 @@
 
 
 def get_audit_logs(timeframe: int) -> Dict:
+    demisto_version: str = get_demisto_version().get("version")
+    demisto.debug(f'{demisto_version=}')
+    if not demisto_version:
+        raise ValueError('Could not get the version of XSOAR')
+
     timefrom = datetime.now() - timedelta(hours=int(timeframe))
     timestring = timefrom.strftime('%Y-%m-%dT%H:%M:%S')
-    parameters = {"uri": "/settings/audits",
-                  "body": {"size": 1000,
-                           "query": f"modified:>{timestring}"}}
-    results = demisto.executeCommand('demisto-api-post', parameters)
+
+    if demisto_version.startswith("6"):  # xsoar 6
+        uri = "/settings/audits"
+        body = {
+            'size': 1000,
+            "query": f"modified:>{timestring}"
+        }
+    else:  # xsoar 8
+        uri = "/public_api/v1/audits/management_logs"
+        body = {
+            "request_data": {
+                "search_to": 1000,
+                "filters": [
+                    {
+                        'field': 'modification_time',
+                        'operator': 'gte',
+                        'value': date_to_timestamp(timestring)
+                    },
+                ]
+            }
+        }
+
+    results = demisto.executeCommand('demisto-api-post', {"uri": uri, "body": body})
     return results[0]['Contents']['response']