Update abuse.ch services

api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py

@@ -4,7 +4,6 @@
 import json
 import logging
 import time
-from typing import Dict
 
 import requests
 
@@ -27,7 +26,10 @@ class YARAifyFileScan(FileAnalyzer, YARAify):
     skip_noisy: bool
     skip_known: bool
 
-    def config(self, runtime_configuration: Dict):
+    def update(self) -> bool:
+        pass
+
+    def config(self, runtime_configuration: dict):
         FileAnalyzer.config(self, runtime_configuration)
         self.query = "lookup_hash"
         YARAify.config(self, runtime_configuration)
@@ -73,7 +75,9 @@ def run(self):
                 "file": (name_to_send, file),
             }
             logger.info(f"yara file scan md5 {self.md5} sending sample for analysis")
-            response = requests.post(self.url, files=files_)
+            response = requests.post(
+                self.url, files=files_, headers=self.authentication_header
+            )
             response.raise_for_status()
             scan_response = response.json()
             scan_query_status = scan_response.get("query_status")
@@ -92,7 +96,9 @@ def run(self):
                             f"task_id: {task_id}"
                         )
                         data = {"query": "get_results", "task_id": task_id}
-                        response = requests.post(self.url, json=data)
+                        response = requests.post(
+                            self.url, json=data, headers=self.authentication_header
+                        )
                         response.raise_for_status()
                         task_response = response.json()
                         logger.debug(task_response)

api_app/analyzers_manager/migrations/0146_analyzer_config_wad.py

@@ -0,0 +1,128 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+    ReverseManyToOneDescriptor,
+    ReverseOneToOneDescriptor,
+)
+
+plugin = {
+    "python_module": {
+        "health_check_schedule": None,
+        "update_schedule": None,
+        "module": "wad.WAD",
+        "base_path": "api_app.analyzers_manager.observable_analyzers",
+    },
+    "name": "WAD",
+    "description": "[WAD](https://github.com/CERN-CERT/WAD) (Web Application Detector) lets you analyze given URL(s) and detect technologies used by web application behind that URL, from the OS and web server level, to the programming platform and frameworks, as well as server- and client-side applications, tools and libraries.",
+    "disabled": False,
+    "soft_time_limit": 60,
+    "routing_key": "default",
+    "health_check_status": True,
+    "type": "observable",
+    "docker_based": False,
+    "maximum_tlp": "CLEAR",
+    "observable_supported": ["url"],
+    "supported_filetypes": [],
+    "run_hash": False,
+    "run_hash_type": "",
+    "not_supported_filetypes": [],
+    "mapping_data_model": {},
+    "model": "analyzers_manager.AnalyzerConfig",
+}
+
+params = []
+
+values = []
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [
+            ForwardManyToOneDescriptor,
+            ReverseManyToOneDescriptor,
+            ReverseOneToOneDescriptor,
+            ForwardOneToOneDescriptor,
+        ]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0065_job_mpnodesearch"),
+        (
+            "analyzers_manager",
+            "0145_analyzer_config_ultradns_malicious_detector",
+        ),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/analyzers_manager/migrations/0147_alter_analyzer_config_feodo_yaraify_urlhaus_yaraify_scan.py

@@ -0,0 +1,98 @@
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+
+    # observables
+    observable_analyzers = [
+        "urlhaus.URLHaus",
+        "yaraify.YARAify",
+        "feodo_tracker.Feodo_Tracker",
+        "threatfox.ThreatFox",
+        "mb_get.MB_GET",
+        "mb_google.MB_GOOGLE",
+    ]
+    for observable_analyzer in observable_analyzers:
+        module = PythonModule.objects.get(
+            module=observable_analyzer,
+            base_path="api_app.analyzers_manager.observable_analyzers",
+        )
+        Parameter.objects.create(
+            name="service_api_key",
+            type="str",
+            description="Optional API key to connect to abuse.ch services.",
+            is_secret=True,
+            required=False,
+            python_module=module,
+        )
+
+    # files
+    yaraify_scan_module = PythonModule.objects.get(
+        module="yaraify_file_scan.YARAifyFileScan",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    Parameter.objects.create(
+        name="service_api_key",
+        type="str",
+        description="Optional API key to connect to abuse.ch services.",
+        is_secret=True,
+        required=False,
+        python_module=yaraify_scan_module,
+    )
+
+
+def reverse_migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+
+    # observables
+    observable_analyzers = [
+        "urlhaus.URLHaus",
+        "yaraify.YARAify",
+        "feodo_tracker.Feodo_Tracker",
+        "threatfox.ThreatFox",
+        "mb_get.MB_GET",
+        "mb_google.MB_GOOGLE",
+    ]
+    for observable_analyzer in observable_analyzers:
+        module = PythonModule.objects.get(
+            module=observable_analyzer,
+            base_path="api_app.analyzers_manager.observable_analyzers",
+        )
+        Parameter.objects.get(
+            name="service_api_key",
+            type="str",
+            description="Optional API key to connect to abuse.ch services.",
+            is_secret=True,
+            required=False,
+            python_module=module,
+        ).delete()
+
+    # files
+    yaraify_scan_module = PythonModule.objects.get(
+        module="yaraify_file_scan.YARAifyFileScan",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    Parameter.objects.get(
+        name="service_api_key",
+        type="str",
+        description="Optional API key to connect to abuse.ch services.",
+        is_secret=True,
+        required=False,
+        python_module=yaraify_scan_module,
+    ).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0065_job_mpnodesearch"),
+        (
+            "analyzers_manager",
+            "0146_analyzer_config_wad",
+        ),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/analyzers_manager/observable_analyzers/feodo_tracker.py

@@ -11,12 +11,14 @@
 
 from api_app.analyzers_manager import classes
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from api_app.mixins import AbuseCHMixin
+from api_app.models import PluginConfig
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
 logger = logging.getLogger(__name__)
 
 
-class Feodo_Tracker(classes.ObservableAnalyzer):
+class Feodo_Tracker(AbuseCHMixin, classes.ObservableAnalyzer):
     """
     Feodo Tracker offers various blocklists,
     helping network owners to protect their
@@ -65,6 +67,20 @@ def run(self):
             raise AnalyzerRunException(f"Key error in run: {e}")
         return result
 
+    @classmethod
+    def get_service_auth_headers(cls) -> {}:
+        for plugin in PluginConfig.objects.filter(
+            parameter__python_module=cls.python_module,
+            parameter__is_secret=True,
+            parameter__name="service_api_key",
+        ):
+            if plugin.value:
+                logger.debug("Found auth key for feodo tracker update")
+                return {"Auth-Key": plugin.value}
+
+        logger.debug("Not found auth key for feodo tracker update")
+        return {}
+
     @classmethod
     def update(cls) -> bool:
         """
@@ -74,7 +90,7 @@ def update(cls) -> bool:
             logger.info(f"starting download of db from {db_url}")
 
             try:
-                r = requests.get(db_url)
+                r = requests.get(db_url, headers=cls.get_service_auth_headers())
                 r.raise_for_status()
             except requests.RequestException:
                 return False

api_app/analyzers_manager/observable_analyzers/mb_get.py

@@ -1,27 +1,40 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
+import logging
 
 import requests
 
 from api_app.analyzers_manager import classes
+from api_app.mixins import AbuseCHMixin
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
+logger = logging.getLogger(__name__)
 
-class MB_GET(classes.ObservableAnalyzer):
+
+class MB_GET(AbuseCHMixin, classes.ObservableAnalyzer):
     url: str = "https://mb-api.abuse.ch/api/v1/"
     sample_url: str = "https://bazaar.abuse.ch/sample/"
 
+    def update(self) -> bool:
+        pass
+
     def run(self):
-        return self.query_mb_api(observable_name=self.observable_name)
+        return self.query_mb_api(
+            observable_name=self.observable_name,
+            headers=self.get_compiled_authentication_header(),
+        )
 
     @classmethod
-    def query_mb_api(cls, observable_name: str) -> dict:
+    def query_mb_api(cls, observable_name: str, headers: dict = None) -> dict:
         """
         This is in a ``classmethod`` so it can be reused in ``MB_GOOGLE``.
         """
         post_data = {"query": "get_info", "hash": observable_name}
 
-        response = requests.post(cls.url, data=post_data)
+        if headers is None:
+            headers = {}
+
+        response = requests.post(cls.url, data=post_data, headers=headers)
         response.raise_for_status()
 
         result = response.json()

api_app/analyzers_manager/observable_analyzers/mb_google.py

@@ -11,13 +11,22 @@ class MB_GOOGLE(MB_GET):
     This is a modified version of MB_GET.
     """
 
+    # API key to access abuse.ch services
+    _service_api_key: str
+
+    def update(self) -> bool:
+        pass
+
     def run(self):
         results = {}
 
         query = f"{self.observable_name} site:bazaar.abuse.ch"
         for url in googlesearch.search(query, stop=20):
             mb_hash = url.split("/")[-2]
-            res = super().query_mb_api(observable_name=mb_hash)
+            res = super().query_mb_api(
+                observable_name=mb_hash,
+                headers=self.get_compiled_authentication_header(),
+            )
             results[mb_hash] = res
 
         return results