Update abuse.ch services

api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py

@@ -26,6 +26,11 @@ class YARAifyFileScan(FileAnalyzer, YARAify):
     share_file: bool
     skip_noisy: bool
     skip_known: bool
+    # API key to access abuse.ch services
+    _service_api_key: str
+
+    def update(self) -> bool:
+        pass
 
     def config(self, runtime_configuration: Dict):
         FileAnalyzer.config(self, runtime_configuration)
@@ -42,6 +47,11 @@ def config(self, runtime_configuration: Dict):
                 "Unable to send file without having api_key_identifier set"
             )
 
+        self.headers = {}
+        if self._service_api_key:
+            logger.debug("Found auth key for YARAify file request")
+            self.headers.setdefault("Auth-Key", self._service_api_key)
+
     def run(self):
         name_to_send = self.filename if self.filename else self.md5
         file = self.read_file_bytes()
@@ -73,7 +83,7 @@ def run(self):
                 "file": (name_to_send, file),
             }
             logger.info(f"yara file scan md5 {self.md5} sending sample for analysis")
-            response = requests.post(self.url, files=files_)
+            response = requests.post(self.url, files=files_, headers=self.headers)
             response.raise_for_status()
             scan_response = response.json()
             scan_query_status = scan_response.get("query_status")
@@ -92,7 +102,9 @@ def run(self):
                             f"task_id: {task_id}"
                         )
                         data = {"query": "get_results", "task_id": task_id}
-                        response = requests.post(self.url, json=data)
+                        response = requests.post(
+                            self.url, json=data, headers=self.headers
+                        )
                         response.raise_for_status()
                         task_response = response.json()
                         logger.debug(task_response)

api_app/analyzers_manager/migrations/0146_analyzer_config_wad.py

@@ -0,0 +1,128 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+    ReverseManyToOneDescriptor,
+    ReverseOneToOneDescriptor,
+)
+
+plugin = {
+    "python_module": {
+        "health_check_schedule": None,
+        "update_schedule": None,
+        "module": "wad.WAD",
+        "base_path": "api_app.analyzers_manager.observable_analyzers",
+    },
+    "name": "WAD",
+    "description": "[WAD](https://github.com/CERN-CERT/WAD) (Web Application Detector) lets you analyze given URL(s) and detect technologies used by web application behind that URL, from the OS and web server level, to the programming platform and frameworks, as well as server- and client-side applications, tools and libraries.",
+    "disabled": False,
+    "soft_time_limit": 60,
+    "routing_key": "default",
+    "health_check_status": True,
+    "type": "observable",
+    "docker_based": False,
+    "maximum_tlp": "CLEAR",
+    "observable_supported": ["url"],
+    "supported_filetypes": [],
+    "run_hash": False,
+    "run_hash_type": "",
+    "not_supported_filetypes": [],
+    "mapping_data_model": {},
+    "model": "analyzers_manager.AnalyzerConfig",
+}
+
+params = []
+
+values = []
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [
+            ForwardManyToOneDescriptor,
+            ReverseManyToOneDescriptor,
+            ReverseOneToOneDescriptor,
+            ForwardOneToOneDescriptor,
+        ]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0065_job_mpnodesearch"),
+        (
+            "analyzers_manager",
+            "0145_analyzer_config_ultradns_malicious_detector",
+        ),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/analyzers_manager/migrations/0147_alter_analyzer_config_feodo_yaraify_urlhaus_yaraify_scan.py

@@ -0,0 +1,98 @@
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+
+    # observables
+    observable_analyzers = [
+        "urlhaus.URLHaus",
+        "yaraify.YARAify",
+        "feodo_tracker.Feodo_Tracker",
+        "threatfox.ThreatFox",
+        "mb_get.MB_GET",
+        "mb_google.MB_GOOGLE",
+    ]
+    for observable_analyzer in observable_analyzers:
+        module = PythonModule.objects.get(
+            module=observable_analyzer,
+            base_path="api_app.analyzers_manager.observable_analyzers",
+        )
+        Parameter.objects.create(
+            name="service_api_key",
+            type="str",
+            description="Optional API key to connect to abuse.ch services.",
+            is_secret=True,
+            required=False,
+            python_module=module,
+        )
+
+    # files
+    yaraify_scan_module = PythonModule.objects.get(
+        module="yaraify_file_scan.YARAifyFileScan",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    Parameter.objects.create(
+        name="service_api_key",
+        type="str",
+        description="Optional API key to connect to abuse.ch services.",
+        is_secret=True,
+        required=False,
+        python_module=yaraify_scan_module,
+    )
+
+
+def reverse_migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+
+    # observables
+    observable_analyzers = [
+        "urlhaus.URLHaus",
+        "yaraify.YARAify",
+        "feodo_tracker.Feodo_Tracker",
+        "threatfox.ThreatFox",
+        "mb_get.MB_GET",
+        "mb_google.MB_GOOGLE",
+    ]
+    for observable_analyzer in observable_analyzers:
+        module = PythonModule.objects.get(
+            module=observable_analyzer,
+            base_path="api_app.analyzers_manager.observable_analyzers",
+        )
+        Parameter.objects.get(
+            name="service_api_key",
+            type="str",
+            description="Optional API key to connect to abuse.ch services.",
+            is_secret=True,
+            required=False,
+            python_module=module,
+        ).delete()
+
+    # files
+    yaraify_scan_module = PythonModule.objects.get(
+        module="yaraify_file_scan.YARAifyFileScan",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    Parameter.objects.get(
+        name="service_api_key",
+        type="str",
+        description="Optional API key to connect to abuse.ch services.",
+        is_secret=True,
+        required=False,
+        python_module=yaraify_scan_module,
+    ).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0065_job_mpnodesearch"),
+        (
+            "analyzers_manager",
+            "0146_analyzer_config_wad",
+        ),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/analyzers_manager/observable_analyzers/feodo_tracker.py

@@ -11,6 +11,7 @@
 
 from api_app.analyzers_manager import classes
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from api_app.models import PluginConfig
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
 logger = logging.getLogger(__name__)
@@ -25,6 +26,8 @@ class Feodo_Tracker(classes.ObservableAnalyzer):
 
     use_recommended_url: bool
     update_on_run: bool = True
+    # API key to access abuse.ch services
+    _service_api_key: str
 
     @classmethod
     @property
@@ -65,6 +68,20 @@ def run(self):
             raise AnalyzerRunException(f"Key error in run: {e}")
         return result
 
+    @classmethod
+    def get_service_auth_headers(cls) -> {}:
+        for plugin in PluginConfig.objects.filter(
+            parameter__python_module=cls.python_module,
+            parameter__is_secret=True,
+            parameter__name="service_api_key",
+        ):
+            if plugin.value:
+                logger.debug("Found auth key for feodo tracker update")
+                return {"Auth-Key": plugin.value}
+
+        logger.debug("Not found auth key for feodo tracker update")
+        return {}
+
     @classmethod
     def update(cls) -> bool:
         """
@@ -74,7 +91,7 @@ def update(cls) -> bool:
             logger.info(f"starting download of db from {db_url}")
 
             try:
-                r = requests.get(db_url)
+                r = requests.get(db_url, headers=cls.get_service_auth_headers())
                 r.raise_for_status()
             except requests.RequestException:
                 return False

api_app/analyzers_manager/observable_analyzers/mb_get.py

@@ -1,27 +1,42 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
+import logging
 
 import requests
 
 from api_app.analyzers_manager import classes
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
+logger = logging.getLogger(__name__)
+
 
 class MB_GET(classes.ObservableAnalyzer):
     url: str = "https://mb-api.abuse.ch/api/v1/"
     sample_url: str = "https://bazaar.abuse.ch/sample/"
+    # API key to access abuse.ch services
+    _service_api_key: str
+
+    def update(self) -> bool:
+        pass
 
     def run(self):
-        return self.query_mb_api(observable_name=self.observable_name)
+        return self.query_mb_api(
+            observable_name=self.observable_name, service_api_key=self._service_api_key
+        )
 
     @classmethod
-    def query_mb_api(cls, observable_name: str) -> dict:
+    def query_mb_api(cls, observable_name: str, service_api_key: str = None) -> dict:
         """
         This is in a ``classmethod`` so it can be reused in ``MB_GOOGLE``.
         """
         post_data = {"query": "get_info", "hash": observable_name}
 
-        response = requests.post(cls.url, data=post_data)
+        headers = {}
+        if service_api_key:
+            logger.debug("Found auth key for MB request")
+            headers.setdefault("Auth-Key", service_api_key)
+
+        response = requests.post(cls.url, data=post_data, headers=headers)
         response.raise_for_status()
 
         result = response.json()

api_app/analyzers_manager/observable_analyzers/mb_google.py

@@ -11,13 +11,21 @@ class MB_GOOGLE(MB_GET):
     This is a modified version of MB_GET.
     """
 
+    # API key to access abuse.ch services
+    _service_api_key: str
+
+    def update(self) -> bool:
+        pass
+
     def run(self):
         results = {}
 
         query = f"{self.observable_name} site:bazaar.abuse.ch"
         for url in googlesearch.search(query, stop=20):
             mb_hash = url.split("/")[-2]
-            res = super().query_mb_api(observable_name=mb_hash)
+            res = super().query_mb_api(
+                observable_name=mb_hash, service_api_key=self._service_api_key
+            )
             results[mb_hash] = res
 
         return results