remove old regexes from content (#7398)

* remove old regexes from content

* use demisto-sdk from master

* Update dev-requirements-py3.txt

Co-authored-by: reut shalem <[email protected]>

* Update dev-requirements-py3.txt

* replace old regexes

* sdk release 1-1-2 test

* sdk release 1-1-2 test

* sdk release 1-1-2 test

* sdk release 1-1-2

Co-authored-by: reut shalem <[email protected]>
Co-authored-by: rshalem <[email protected]>

Author: anara123

Tests/configure_and_test_integration_instances.py

@@ -15,14 +15,12 @@
 from demisto_sdk.commands.common.tools import print_error, print_warning, print_color, LOG_COLORS, run_threads_list, \
     run_command, get_last_release_version, checked_type, get_yaml, str2bool, server_version_compare
 from demisto_sdk.commands.validate.file_validator import FilesValidator
-from demisto_sdk.commands.common.constants import YML_INTEGRATION_REGEXES, INTEGRATION_REGEX, PACKS_INTEGRATION_REGEX, \
-    BETA_INTEGRATION_REGEX, RUN_ALL_TESTS_FORMAT
+from demisto_sdk.commands.common.constants import YML_INTEGRATION_REGEXES, RUN_ALL_TESTS_FORMAT
 
 from Tests.test_integration import __get_integration_config, __test_integration_instance, \
     __disable_integrations_instances
 from Tests.test_content import load_conf_files, extract_filtered_tests, ParallelPrintsManager, \
     get_server_numeric_version
-
 from Tests.update_content_data import update_content
 from Tests.Marketplace.search_and_install_packs import search_and_install_packs_and_their_dependencies
 
@@ -198,7 +196,6 @@ def get_new_and_modified_integration_files(git_sha1):
     change_log = run_command('git diff --name-status {}'.format(git_sha1))
     modified_files, added_files, _, _ = file_validator.get_modified_files(change_log, tag)
     all_integration_regexes = YML_INTEGRATION_REGEXES
-    all_integration_regexes.extend([INTEGRATION_REGEX, PACKS_INTEGRATION_REGEX, BETA_INTEGRATION_REGEX])
 
     new_integration_files = [
         file_path for file_path in added_files if checked_type(file_path, all_integration_regexes)

Tests/scripts/collect_tests_and_content_packs.py

@@ -4,7 +4,6 @@
 Overview can be found at: https://confluence.paloaltonetworks.com/display/DemistoContent/Configure+Test+Filter
 """
 import os
-import re
 import sys
 import json
 import glob
@@ -34,18 +33,16 @@
 
 TEST_DATA_INTEGRATION_YML_REGEX = r'Tests\/scripts\/infrastructure_tests\/tests_data\/mock_integrations\/.*\.yml'
 INTEGRATION_REGEXES = [
-    INTEGRATION_REGEX,
-    BETA_INTEGRATION_REGEX,
-    PACKS_INTEGRATION_REGEX,
+    PACKS_INTEGRATION_PY_REGEX,
+    PACKS_INTEGRATION_PS_TEST_REGEX,
     TEST_DATA_INTEGRATION_YML_REGEX
 ]
 TEST_DATA_SCRIPT_YML_REGEX = r'Tests/scripts/infrastructure_tests/tests_data/mock_scripts/.*.yml'
 SCRIPT_REGEXES = [
     TEST_DATA_SCRIPT_YML_REGEX
 ]
 INCIDENT_FIELD_REGEXES = [
-    INCIDENT_FIELD_REGEX,
-    PACKS_INCIDENT_FIELDS_REGEX
+    PACKS_INCIDENT_FIELD_JSON_REGEX
 ]
 FILES_IN_SCRIPTS_OR_INTEGRATIONS_DIRS_REGEXES = [
     FILE_IN_INTEGRATIONS_DIR_REGEX,
@@ -55,20 +52,19 @@
 ]
 CHECKED_TYPES_REGEXES = [
     # Integrations
-    INTEGRATION_REGEX,
-    INTEGRATION_YML_REGEX,
-    BETA_INTEGRATION_REGEX,
-    PACKS_INTEGRATION_REGEX,
+    PACKS_INTEGRATION_PY_REGEX,
     PACKS_INTEGRATION_YML_REGEX,
+    PACKS_INTEGRATION_NON_SPLIT_YML_REGEX,
+    PACKS_INTEGRATION_PS_REGEX,
+
     # Scripts
-    SCRIPT_REGEX,
-    SCRIPT_YML_REGEX,
     PACKS_SCRIPT_REGEX,
     PACKS_SCRIPT_YML_REGEX,
+    PACKS_SCRIPT_NON_SPLIT_YML_REGEX,
+
     # Playbooks
     PLAYBOOK_REGEX,
-    BETA_PLAYBOOK_REGEX,
-    PACKS_PLAYBOOK_YML_REGEX
+    PLAYBOOK_YML_REGEX
 ]
 
 # File names
@@ -159,8 +155,7 @@ def get_modified_files(files_string):
             # reputations.json
             elif re.match(INDICATOR_TYPES_REPUTATIONS_REGEX, file_path, re.IGNORECASE) or \
                     re.match(PACKS_INDICATOR_TYPES_REPUTATIONS_REGEX, file_path, re.IGNORECASE) or \
-                    re.match(INDICATOR_TYPES_REGEX, file_path, re.IGNORECASE) or \
-                    re.match(PACKS_INDICATOR_TYPES_REGEX, file_path, re.IGNORECASE):
+                    re.match(PACKS_INDICATOR_TYPE_JSON_REGEX, file_path, re.IGNORECASE):
                 is_reputations_json = True
 
             elif checked_type(file_path, INCIDENT_FIELD_REGEXES):
@@ -870,7 +865,7 @@ def get_test_conf_from_conf(test_id, server_version, conf=None):
     # return None if nothing is found
     test_conf = next((test_conf for test_conf in test_conf_lst if (
         test_conf.get('playbookID') == test_id
-        and is_runnable_in_server_version(from_v=test_conf.get('fromversion', '0'),
+        and is_runnable_in_server_version(from_v=test_conf.get('fromversion', '0.0'),
                                           server_v=server_version,
                                           to_v=test_conf.get('toversion', '99.99.99'))
     )), None)
@@ -894,7 +889,7 @@ def extract_matching_object_from_id_set(obj_id, obj_set, server_version='0'):
                 continue
 
         # check if object is runnable
-        fromversion = obj.get('fromversion', '0')
+        fromversion = obj.get('fromversion', '0.0')
         toversion = obj.get('toversion', '99.99.99')
         if is_runnable_in_server_version(from_v=fromversion, server_v=server_version, to_v=toversion):
             return obj
@@ -962,7 +957,7 @@ def is_test_runnable(test_id, id_set, conf, server_version):
     if not test_conf:
         print_warning(f'{warning_prefix} - couldn\'t find test in conf.json')
         return False
-    conf_fromversion = test_conf.get('fromversion', '0')
+    conf_fromversion = test_conf.get('fromversion', '0.0')
     conf_toversion = test_conf.get('toversion', '99.99.99')
     test_playbooks_set = id_set.get('TestPlaybooks', [])
     test_playbook_obj = extract_matching_object_from_id_set(test_id, test_playbooks_set, server_version)

Tests/scripts/validate.sh

@@ -1,7 +1,11 @@
 #!/usr/bin/env bash
 set -e
 
-if [ "${CHECK_BACKWARD}" = "true" ] ;
+if [[ $CIRCLE_BRANCH = master ]];
+  then
+    demisto-sdk validate -a
+
+elif [ "${CHECK_BACKWARD}" = "true" ] ;
   then
      demisto-sdk validate -g --post-commit
 

dev-requirements-py3.txt

@@ -1,7 +1,7 @@
 flake8==3.7.9
 bandit==1.6.2
 demisto-py==2.0.11
-demisto-sdk==1.1.1
+demisto-sdk==1.1.2
 pytest==5.2.1
 pytest-mock==1.11.1
 requests-mock==1.7.0
@@ -14,4 +14,4 @@ beautifulsoup4==4.8.1
 pypdf2==1.26.0
 freezegun==0.3.12
 vulture==1.5
-git+https://github.com/coyo8/parinx.git@6493798ceba8089345d970f71be4a896eb6b081d
+git+https://github.com/coyo8/parinx.git@6493798ceba8089345d970f71be4a896eb6b081d