Add a manual fatch once in 12 hours (demisto#31123)

* fixes

* http module

* CSV

* common server

* tests

* RN

* link

* RN

* change RN

* one more

* pre commit

* update base version

* [known_words]

* removing typing

* swap the known words

* RN

* fix RN

* Bump pack from version FeedMalwareBazaar to 1.0.30.

* Bump pack from version AccentureCTI_Feed to 1.1.27.

* Bump pack from version FeedGCPWhitelist to 2.0.30.

* Bump pack from version Base to 1.32.52.

* make it better

* docs

* CR

* cr

* Fixing dirty merge #1

* fixing dirty merge #2

* fix dirty merge #3

* more

* fox dirty merge #4

* common

* poetry

* fix dirty merge #5

* fix test date

* base rn

* RN

* fix common docstring

* fix rn

* fix errors in build

* shirley

* Bump pack from version Base to 1.32.54.

* RN

* mypy

* fix common server

* ignore type error

* skip test

* fix test name

* add import

* remove the import, test is failing

* fixed function and test

* space

* conf

* add a test for a uniq time zone

* fix test

* move the import into the function

* move the import from the test as well

* replace timezone with pytz, to fit python 2

* Bump pack from version Base to 1.33.1.

* fix test comment

---------

Co-authored-by: Content Bot <[email protected]>

Author: RosenbergYehuda

Packs/AccentureCTI_Feed/ReleaseNotes/1_1_28.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### ACTI Indicator Feed
+
+Fixed an issue where indicators were expiring due to prolonged periods of inactivity in the data source by implementing a solution that enforces a bi-daily update for existing indicators, even if the corresponding resource hasn't been updated.

Packs/AccentureCTI_Feed/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Accenture CTI Feed",
     "description": "Accenture Cyber Threat Intelligence Feed",
     "support": "partner",
-    "currentVersion": "1.1.27",
+    "currentVersion": "1.1.28",
     "author": "Accenture",
     "url": "https://www.accenture.com/us-en/services/security/cyber-defense",
     "email": "[email protected]",

Packs/ApiModules/Scripts/CSVFeedApiModule/CSVFeedApiModule.py

@@ -13,6 +13,7 @@
 
 # Globals
 DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+THRESHOLD_IN_SECONDS = 43200    # 12 hours in seconds
 
 
 class Client(BaseClient):
@@ -140,6 +141,15 @@ def build_iterator(self, **kwargs):
                 last_run = demisto.getLastRun()
                 etag = last_run.get(url, {}).get('etag')
                 last_modified = last_run.get(url, {}).get('last_modified')
+                last_updated = last_run.get(url, {}).get('last_updated')
+                # To avoid issues with indicators expiring, if 'last_updated' is over X hours old,
+                # we'll refresh the indicators to ensure their expiration time is updated.
+                # For further details, refer to : https://confluence-dc.paloaltonetworks.com/display/DemistoContent/Json+Api+Module     # noqa: E501
+                if last_updated and has_passed_time_threshold(timestamp_str=last_updated, seconds_threshold=THRESHOLD_IN_SECONDS):
+                    last_modified = None
+                    etag = None
+                    demisto.debug("Since it's been a long time with no update, to make sure we are keeping the indicators alive, \
+                        we will refetch them from scratch")
 
                 if etag:
                     self.headers['If-None-Match'] = etag
@@ -250,14 +260,17 @@ def get_no_update_value(response: requests.models.Response, url: str) -> bool:
 
     etag = response.headers.get('ETag')
     last_modified = response.headers.get('Last-Modified')
+    current_time = datetime.utcnow()
+    # Save the current time as the last updated time. This will be used to indicate the last time the feed was updated in XSOAR.
+    last_updated = current_time.strftime(DATE_FORMAT)
 
     if not etag and not last_modified:
         demisto.debug('Last-Modified and Etag headers are not exists,'
                       'createIndicators will be executed with noUpdate=False.')
         return False
 
     last_run = demisto.getLastRun()
-    last_run[url] = {'last_modified': last_modified, 'etag': etag}
+    last_run[url] = {'last_modified': last_modified, 'etag': etag, 'last_updated': last_updated}
     demisto.setLastRun(last_run)
 
     demisto.debug('New indicators fetched - the Last-Modified value has been updated,'
@@ -344,7 +357,7 @@ def fetch_indicators_command(client: Client, default_indicator_type: str, auto_d
     config = client.feed_url_to_config or {}
 
     # set noUpdate flag in createIndicators command True only when all the results from all the urls are True.
-    no_update = all([next(iter(item.values())).get('no_update', False) for item in iterator])
+    no_update = all(next(iter(item.values())).get('no_update', False) for item in iterator)
 
     for url_to_reader in iterator:
         for url, reader in url_to_reader.items():

Packs/ApiModules/Scripts/CSVFeedApiModule/CSVFeedApiModule_test.py

@@ -1,6 +1,5 @@
 import requests_mock
 from CSVFeedApiModule import *
-import io
 import pytest
 
 
@@ -243,7 +242,7 @@ def test_tags_not_exists(self):
 
 
 def util_load_json(path):
-    with io.open(path, mode='r', encoding='utf-8') as f:
+    with open(path, 'r', encoding='utf-8') as f:
         return json.loads(f.read())
 
 
@@ -504,3 +503,36 @@ def test_build_iterator_modified_headers(mocker):
     result = client.build_iterator()
     assert 'Authorization' in mock_session.call_args[0][0].headers
     assert result
+
+
+@pytest.mark.parametrize('has_passed_time_threshold_response, expected_result', [
+    (True, {}),
+    (False, {'If-None-Match': 'etag', 'If-Modified-Since': '2023-05-29T12:34:56Z'})
+])
+def test_build_iterator__with_and_without_passed_time_threshold(mocker, has_passed_time_threshold_response, expected_result):
+    """
+    Given
+    - A boolean result from the has_passed_time_threshold function
+    When
+    - Running build_iterator method.
+    Then
+    - Ensure the next request headers will be as expected:
+        case 1: has_passed_time_threshold_response is True, no headers will be added
+        case 2: has_passed_time_threshold_response is False, headers containing 'last_modified' and 'etag' will be added
+    """
+    mocker.patch('CommonServerPython.get_demisto_version', return_value={"version": "6.5.0"})
+    mock_session = mocker.patch.object(requests.Session, 'send')
+    mocker.patch('CSVFeedApiModule.has_passed_time_threshold', return_value=has_passed_time_threshold_response)
+    mocker.patch('demistomock.getLastRun', return_value={
+        'https://api.github.com/meta': {
+            'etag': 'etag',
+            'last_modified': '2023-05-29T12:34:56Z',
+            'last_updated': '2023-05-05T09:09:06Z'
+        }})
+    client = Client(
+        url='https://api.github.com/meta',
+        credentials={'identifier': 'user', 'password': 'password'})
+
+    client.build_iterator()
+    assert mock_session.call_args[0][0].headers.get('If-None-Match') == expected_result.get('If-None-Match')
+    assert mock_session.call_args[0][0].headers.get('If-Modified-Since') == expected_result.get('If-Modified-Since')

Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.py

@@ -14,6 +14,7 @@
 TAGS = 'tags'
 TLP_COLOR = 'trafficlightprotocol'
 DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+THRESHOLD_IN_SECONDS = 43200        # 12 hours in seconds
 
 
 class Client(BaseClient):
@@ -216,6 +217,16 @@ def build_iterator(self, **kwargs):
                     last_run = demisto.getLastRun()
                     etag = last_run.get(url, {}).get('etag')
                     last_modified = last_run.get(url, {}).get('last_modified')
+                    last_updated = last_run.get(url, {}).get('last_updated')
+                    # To avoid issues with indicators expiring, if 'last_updated' is over X hours old,
+                    # we'll refresh the indicators to ensure their expiration time is updated.
+                    # For further details, refer to : https://confluence-dc.paloaltonetworks.com/display/DemistoContent/Json+Api+Module     # noqa: E501
+                    if last_updated and has_passed_time_threshold(timestamp_str=last_updated,
+                                                                  seconds_threshold=THRESHOLD_IN_SECONDS):
+                        last_modified = None
+                        etag = None
+                        demisto.debug("Since it's been a long time with no update, to make sure we are keeping the indicators\
+                            alive, we will refetch them from scratch")
                     if etag:
                         if not kwargs.get('headers'):
                             kwargs['headers'] = {}
@@ -269,15 +280,9 @@ def build_iterator(self, **kwargs):
                 lines = res_data.get('response')
                 result = lines.iter_lines()
                 if self.encoding is not None:
-                    result = map(
-                        lambda x: x.decode(self.encoding).encode('utf_8'),
-                        result
-                    )
+                    result = (x.decode(self.encoding).encode('utf_8') for x in result)
                 else:
-                    result = map(
-                        lambda x: x.decode('utf_8'),
-                        result
-                    )
+                    result = (x.decode('utf_8') for x in result)
                 if self.ignore_regex is not None:
                     result = filter(
                         lambda x: self.ignore_regex.match(x) is None,  # type: ignore[union-attr, arg-type]
@@ -288,8 +293,8 @@ def build_iterator(self, **kwargs):
 
     def custom_fields_creator(self, attributes: dict):
         created_custom_fields = {}
-        for attribute in attributes.keys():
-            if attribute in self.custom_fields_mapping.keys() or attribute in [TAGS, TLP_COLOR]:
+        for attribute in attributes:
+            if attribute in self.custom_fields_mapping or attribute in [TAGS, TLP_COLOR]:
                 if attribute in [TAGS, TLP_COLOR]:
                     created_custom_fields[attribute] = attributes[attribute]
                 else:
@@ -317,14 +322,17 @@ def get_no_update_value(response: requests.Response, url: str) -> bool:
 
     etag = response.headers.get('ETag')
     last_modified = response.headers.get('Last-Modified')
+    current_time = datetime.utcnow()
+    # Save the current time as the last updated time. This will be used to indicate the last time the feed was updated in XSOAR.
+    last_updated = current_time.strftime(DATE_FORMAT)
 
     if not etag and not last_modified:
         demisto.debug('Last-Modified and Etag headers are not exists,'
                       'createIndicators will be executed with noUpdate=False.')
         return False
 
     last_run = demisto.getLastRun()
-    last_run[url] = {'last_modified': last_modified, 'etag': etag}
+    last_run[url] = {'last_modified': last_modified, 'etag': etag, 'last_updated': last_updated}
     demisto.setLastRun(last_run)
 
     demisto.debug('New indicators fetched - the Last-Modified value has been updated,'
@@ -357,13 +365,12 @@ def get_indicator_fields(line, url, feed_tags: list, tlp_color: Optional[str], c
     indicator = None
     fields_to_extract = []
     feed_config = client.feed_url_to_config.get(url, {})
-    if feed_config:
-        if 'indicator' in feed_config:
-            indicator = feed_config['indicator']
-            if 'regex' in indicator:
-                indicator['regex'] = re.compile(indicator['regex'])
-            if 'transform' not in indicator:
-                indicator['transform'] = r'\g<0>'
+    if feed_config and 'indicator' in feed_config:
+        indicator = feed_config['indicator']
+        if 'regex' in indicator:
+            indicator['regex'] = re.compile(indicator['regex'])
+        if 'transform' not in indicator:
+            indicator['transform'] = r'\g<0>'
 
     if 'fields' in feed_config:
         fields = feed_config['fields']
@@ -418,17 +425,17 @@ def fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, c
     indicators = []
 
     # set noUpdate flag in createIndicators command True only when all the results from all the urls are True.
-    no_update = all([next(iter(iterator.values())).get('no_update', False) for iterator in iterators])
+    no_update = all(next(iter(iterator.values())).get('no_update', False) for iterator in iterators)
 
     for iterator in iterators:
         for url, lines in iterator.items():
             for line in lines.get('result', []):
                 attributes, value = get_indicator_fields(line, url, feed_tags, tlp_color, client)
                 if value:
-                    if 'lastseenbysource' in attributes.keys():
+                    if 'lastseenbysource' in attributes:
                         attributes['lastseenbysource'] = datestring_to_server_format(attributes['lastseenbysource'])
 
-                    if 'firstseenbysource' in attributes.keys():
+                    if 'firstseenbysource' in attributes:
                         attributes['firstseenbysource'] = datestring_to_server_format(attributes['firstseenbysource'])
                     indicator_type = determine_indicator_type(
                         client.feed_url_to_config.get(url, {}).get('indicator_type'), itype, auto_detect, value)
@@ -450,7 +457,7 @@ def fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, c
                             relationships_of_indicator = [relationships_lst.to_indicator()]
                             indicator_data['relationships'] = relationships_of_indicator
 
-                    if len(client.custom_fields_mapping.keys()) > 0 or TAGS in attributes.keys():
+                    if len(client.custom_fields_mapping.keys()) > 0 or TAGS in attributes:
                         custom_fields = client.custom_fields_creator(attributes)
                         indicator_data["fields"] = custom_fields
 

Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule_test.py

@@ -2,6 +2,8 @@
     fetch_indicators_command, get_no_update_value
 import requests_mock
 import demistomock as demisto
+import pytest
+import requests
 
 
 def test_get_indicators():
@@ -145,12 +147,12 @@ def test_datestring_to_server_format():
     datestring4 = "2020-02-10T13:39:14.123"
     datestring5 = "2020-02-10T13:39:14Z"
     datestring6 = "2020-11-01T04:16:13-04:00"
-    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring1)
-    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring2)
-    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring3)
-    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring4)
-    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring5)
-    assert '2020-11-01T08:16:13Z' == datestring_to_server_format(datestring6)
+    assert datestring_to_server_format(datestring1) == '2020-02-10T13:39:14Z'
+    assert datestring_to_server_format(datestring2) == '2020-02-10T13:39:14Z'
+    assert datestring_to_server_format(datestring3) == '2020-02-10T13:39:14Z'
+    assert datestring_to_server_format(datestring4) == '2020-02-10T13:39:14Z'
+    assert datestring_to_server_format(datestring5) == '2020-02-10T13:39:14Z'
+    assert datestring_to_server_format(datestring6) == '2020-11-01T08:16:13Z'
 
 
 def test_get_feed_config():
@@ -544,3 +546,35 @@ class MockResponse:
     assert not no_update
     assert demisto.debug.call_args[0][0] == 'Last-Modified and Etag headers are not exists,' \
                                             'createIndicators will be executed with noUpdate=False.'
+
+
+@pytest.mark.parametrize('has_passed_time_threshold_response, expected_result', [
+    (True, None),
+    (False, {'If-None-Match': 'etag', 'If-Modified-Since': '2023-05-29T12:34:56Z'})
+])
+def test_build_iterator__with_and_without_passed_time_threshold(mocker, has_passed_time_threshold_response, expected_result):
+    """
+    Given
+    - A boolean result from the has_passed_time_threshold function
+    When
+    - Running build_iterator method.
+    Then
+    - Ensure the next request headers will be as expected:
+        case 1: has_passed_time_threshold_response is True, no headers will be added
+        case 2: has_passed_time_threshold_response is False, headers containing 'last_modified' and 'etag' will be added
+    """
+    mocker.patch('CommonServerPython.get_demisto_version', return_value={"version": "6.5.0"})
+    mock_session = mocker.patch.object(requests, 'get')
+    mocker.patch('HTTPFeedApiModule.has_passed_time_threshold', return_value=has_passed_time_threshold_response)
+    mocker.patch('demistomock.getLastRun', return_value={
+        'https://api.github.com/meta': {
+            'etag': 'etag',
+            'last_modified': '2023-05-29T12:34:56Z',
+            'last_updated': '2023-05-05T09:09:06Z'
+        }})
+    client = Client(
+        url='https://api.github.com/meta',
+        credentials={'identifier': 'user', 'password': 'password'})
+
+    client.build_iterator()
+    assert mock_session.call_args[1].get('headers') == expected_result

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py

@@ -9,6 +9,9 @@
 # disable insecure warnings
 urllib3.disable_warnings()
 
+DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+THRESHOLD_IN_SECONDS = 43200        # 12 hours in seconds
+
 
 class Client:
     def __init__(self, url: str = '', credentials: dict = None,
@@ -81,7 +84,7 @@ def __init__(self, url: str = '', credentials: dict = None,
 
         if isinstance(self.post_data, str):
             content_type_header = 'Content-Type'
-            if content_type_header.lower() not in [k.lower() for k in self.headers.keys()]:
+            if content_type_header.lower() not in [k.lower() for k in self.headers]:
                 self.headers[content_type_header] = 'application/x-www-form-urlencoded'
 
     @staticmethod
@@ -118,6 +121,15 @@ def build_iterator(self, feed: dict, feed_name: str, **kwargs) -> Tuple[List, bo
             last_run = demisto.getLastRun()
             etag = last_run.get(prefix_feed_name, {}).get('etag') or last_run.get(feed_name, {}).get('etag')
             last_modified = last_run.get(prefix_feed_name, {}).get('last_modified') or last_run.get(feed_name, {}).get('last_modified')  # noqa: E501
+            last_updated = last_run.get(prefix_feed_name, {}).get('last_updated') or last_run.get(feed_name, {}).get('last_updated')  # noqa: E501
+            # To avoid issues with indicators expiring, if 'last_updated' is over X hours old,
+            # we'll refresh the indicators to ensure their expiration time is updated.
+            # For further details, refer to : https://confluence-dc.paloaltonetworks.com/display/DemistoContent/Json+Api+Module
+            if last_updated and has_passed_time_threshold(timestamp_str=last_updated, seconds_threshold=THRESHOLD_IN_SECONDS):
+                last_modified = None
+                etag = None
+                demisto.debug("Since it's been a long time with no update, to make sure we are keeping the indicators alive, \
+                    we will refetch them from scratch")
 
             if etag:
                 self.headers['If-None-Match'] = etag
@@ -180,6 +192,9 @@ def get_no_update_value(response: requests.Response, feed_name: str) -> bool:
 
     etag = response.headers.get('ETag')
     last_modified = response.headers.get('Last-Modified')
+    current_time = datetime.utcnow()
+    # Save the current time as the last updated time. This will be used to indicate the last time the feed was updated in XSOAR.
+    last_updated = current_time.strftime(DATE_FORMAT)
 
     if not etag and not last_modified:
         demisto.debug('Last-Modified and Etag headers are not exists,'
@@ -189,7 +204,8 @@ def get_no_update_value(response: requests.Response, feed_name: str) -> bool:
     last_run = demisto.getLastRun()
     last_run[feed_name] = {
         'last_modified': last_modified,
-        'etag': etag
+        'etag': etag,
+        'last_updated': last_updated
     }
     demisto.setLastRun(last_run)
     demisto.debug(f'JSON: The new last run is: {last_run}')