Custom Analyzer from the GUI

.github/workflows/pull_request_automation.yml

@@ -123,7 +123,7 @@ jobs:
       - name: Set up NodeJS
         uses: actions/setup-node@v4
         with:
-          node-version: 15
+          node-version: 18
       - name: Cache node modules
         uses: actions/cache@v4
         with:

api_app/analyzers_manager/constants.py

@@ -83,3 +83,11 @@ class AllTypes(models.TextChoices):
     HASH = "hash"
     GENERIC = "generic"
     FILE = "file"
+
+
+class HTTPMethods(models.TextChoices):
+    GET = "get"
+    POST = "post"
+    PUT = "put"
+    PATCH = "patch"
+    DELETE = "delete"

api_app/analyzers_manager/migrations/0123_basic_observable_analyzer.py

@@ -0,0 +1,87 @@
+from django.db import migrations
+
+
+def migrate_python_module_pivot(apps, schema_editor):
+    PythonModule = apps.get_model("api_app", "PythonModule")
+    pm, _ = PythonModule.objects.update_or_create(
+        module="basic_observable_analyzer.BasicObservableAnalyzer",
+        base_path="api_app.analyzers_manager.observable_analyzers",
+    )
+    Parameter = apps.get_model("api_app", "Parameter")
+    Parameter.objects.get_or_create(
+        name="url",
+        type="str",
+        python_module=pm,
+        is_secret=False,
+        required=True,
+        defaults={
+            "description": "URL of the instance you want to connect to",
+        },
+    )
+    Parameter.objects.get_or_create(
+        name="api_key_name",
+        type="str",
+        python_module=pm,
+        is_secret=True,
+        required=False,
+        defaults={
+            "description": "API key required for authentication",
+        },
+    )
+    Parameter.objects.get_or_create(
+        name="headers",
+        type="dict",
+        python_module=pm,
+        is_secret=False,
+        required=False,
+        defaults={
+            "description": "Headers used for the request",
+        },
+    )
+    Parameter.objects.get_or_create(
+        name="http_method",
+        type="str",
+        python_module=pm,
+        is_secret=False,
+        required=True,
+        defaults={
+            "description": "HTTP method used for the request",
+        },
+    )
+    Parameter.objects.get_or_create(
+        name="params",
+        type="dict",
+        python_module=pm,
+        is_secret=False,
+        required=False,
+        defaults={
+            "description": "Params used for the query string or request payload",
+        },
+    )
+    Parameter.objects.get_or_create(
+        name="certificate",
+        type="str",
+        python_module=pm,
+        is_secret=True,
+        required=False,
+        defaults={
+            "description": "Instance SSL certificate (multiline string).",
+        },
+    )
+
+
+def reverse_migrate_module_pivot(apps, schema_editor):
+    PythonModule = apps.get_model("api_app", "PythonModule")
+    PythonModule.objects.get(
+        module="basic_observable_analyzer.BasicObservableAnalyzer",
+        base_path="api_app.analyzers_manager.observable_analyzers",
+    ).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analyzers_manager", "0122_alter_soft_time_limit"),
+    ]
+    operations = [
+        migrations.RunPython(migrate_python_module_pivot, reverse_migrate_module_pivot)
+    ]

api_app/analyzers_manager/observable_analyzers/basic_observable_analyzer.py

@@ -0,0 +1,105 @@
+import base64
+import logging
+from tempfile import NamedTemporaryFile
+
+import requests
+
+from api_app.analyzers_manager.classes import ObservableAnalyzer
+from api_app.analyzers_manager.constants import HTTPMethods
+from api_app.analyzers_manager.exceptions import (
+    AnalyzerConfigurationException,
+    AnalyzerRunException,
+)
+from tests.mock_utils import MockUpResponse, if_mock_connections, patch
+
+logger = logging.getLogger(__name__)
+
+
+class BasicObservableAnalyzer(ObservableAnalyzer):
+    url: str
+    headers: dict
+    params: dict
+    _certificate: str
+    _api_key_name: str
+    http_method: str = "get"
+
+    @staticmethod
+    def _clean_certificate(cert):
+        return (
+            cert.replace("-----BEGIN CERTIFICATE-----", "-----BEGIN_CERTIFICATE-----")
+            .replace("-----END CERTIFICATE-----", "-----END_CERTIFICATE-----")
+            .replace(" ", "\n")
+            .replace("-----BEGIN_CERTIFICATE-----", "-----BEGIN CERTIFICATE-----")
+            .replace("-----END_CERTIFICATE-----", "-----END CERTIFICATE-----")
+        )
+
+    def update(self) -> bool:
+        pass
+
+    def run(self):
+        if not hasattr(self, "url"):
+            raise AnalyzerConfigurationException("Instance URL is required")
+        if self.http_method not in HTTPMethods.values:
+            raise AnalyzerConfigurationException("Http method is not valid")
+
+        # replace <observable> placheholder
+        for key in self.params.keys():
+            if self.params[key] == "<observable>":
+                self.params[key] = self.observable_name
+
+        # optional authentication
+        if hasattr(self, "_api_key_name") and self._api_key_name:
+            api_key = self._api_key_name
+            if (
+                "Authorization" in self.headers.keys()
+                and self.headers["Authorization"].split(" ")[0] == "Basic"
+            ):
+                # the API uses basic auth so we need to base64 encode the auth payload
+                api_key = base64.b64encode(self._api_key_name.encode()).decode()
+            # replace <api_key> placeholder
+            for key in self.headers.keys():
+                self.headers[key] = self.headers[key].replace("<api_key>", api_key)
+
+        # optional certificate
+        verify = True  # defualt
+        if hasattr(self, "_certificate") and self._certificate:
+            self.__cert_file = NamedTemporaryFile(mode="w")
+            self.__cert_file.write(self._clean_certificate(self._certificate))
+            self.__cert_file.flush()
+            verify = self.__cert_file.name
+
+        try:
+            if self.http_method == HTTPMethods.GET:
+                url = self.url
+                if not self.params.keys():
+                    url = self.url + self.observable_name
+                response = requests.get(
+                    url,
+                    params=self.params,
+                    headers=self.headers,
+                    verify=verify,
+                )
+            else:
+                request_method = getattr(requests, self.http_method)
+                response = request_method(
+                    self.url, headers=self.headers, json=self.params, verify=verify
+                )
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise AnalyzerRunException(e)
+
+        response_json = response.json()
+        logger.debug(f"response received: {response_json}")
+        return response_json
+
+    @classmethod
+    def _monkeypatch(cls):
+        patches = [
+            if_mock_connections(
+                patch(
+                    "requests.get",
+                    return_value=MockUpResponse({}, 200),
+                ),
+            )
+        ]
+        return super()._monkeypatch(patches=patches)

api_app/analyzers_manager/serializers.py

@@ -1,6 +1,10 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
+from rest_framework import serializers as rfs
+
+from ..models import PluginConfig, PythonModule
 from ..serializers.plugin import (
+    PluginConfigSerializer,
     PythonConfigSerializer,
     PythonConfigSerializerForMigration,
 )
@@ -23,11 +27,50 @@ class Meta:
 
 
 class AnalyzerConfigSerializer(PythonConfigSerializer):
+    plugin_config = rfs.ListField(
+        child=rfs.DictField(), write_only=True, required=False
+    )
+    python_module = rfs.SlugRelatedField(
+        queryset=PythonModule.objects.all(), slug_field="module"
+    )
+
     class Meta:
         model = AnalyzerConfig
         exclude = PythonConfigSerializer.Meta.exclude
         list_serializer_class = PythonConfigSerializer.Meta.list_serializer_class
 
+    def create(self, validated_data):
+        plugin_config = validated_data.pop("plugin_config", {})
+        pc = super().create(validated_data)
+
+        # create plugin config
+        for config in plugin_config:
+            plugin_config_serializer = PluginConfigSerializer(
+                data=config, context={"request": self.context["request"]}
+            )
+            plugin_config_serializer.is_valid(raise_exception=True)
+            plugin_config_serializer.save()
+        return pc
+
+    def update(self, instance, validated_data):
+        plugin_config = validated_data.pop("plugin_config", [])
+        pc = super().update(instance, validated_data)
+
+        # update plugin config
+        for config in plugin_config:
+            plugin_config_serializer = PluginConfigSerializer(
+                data=config, context={"request": self.context["request"]}
+            )
+            plugin_config_serializer.is_valid(raise_exception=True)
+            PluginConfig.objects.filter(
+                owner=self.context["request"].user,
+                analyzer_config=plugin_config_serializer.validated_data[
+                    "analyzer_config"
+                ],
+                parameter=plugin_config_serializer.validated_data["parameter"],
+            ).update_or_create(plugin_config_serializer.validated_data)
+        return pc
+
 
 class AnalyzerConfigSerializerForMigration(PythonConfigSerializerForMigration):
     class Meta:

api_app/analyzers_manager/views.py

@@ -2,9 +2,12 @@
 # See the file 'LICENSE' for copying permission.
 import logging
 
+from rest_framework import mixins
+
+from ..permissions import isPluginActionsPermission
 from ..views import PythonConfigViewSet, PythonReportActionViewSet
 from .filters import AnalyzerConfigFilter
-from .models import AnalyzerReport
+from .models import AnalyzerConfig, AnalyzerReport
 from .serializers import AnalyzerConfigSerializer
 
 logger = logging.getLogger(__name__)
@@ -16,9 +19,21 @@
 ]
 
 
-class AnalyzerConfigViewSet(PythonConfigViewSet):
+class AnalyzerConfigViewSet(
+    PythonConfigViewSet,
+    mixins.CreateModelMixin,
+    mixins.UpdateModelMixin,
+    mixins.DestroyModelMixin,
+):
     serializer_class = AnalyzerConfigSerializer
     filterset_class = AnalyzerConfigFilter
+    queryset = AnalyzerConfig.objects.all()
+
+    def get_permissions(self):
+        permissions = super().get_permissions()
+        if self.action in ["destroy", "update", "partial_update"]:
+            permissions.append(isPluginActionsPermission())
+        return permissions
 
 
 class AnalyzerActionViewSet(PythonReportActionViewSet):

api_app/permissions.py

@@ -100,3 +100,13 @@ def has_object_permission(request, view, obj):
                 and obj_owner.membership.organization
                 == request.user.membership.organization
             )
+
+
+class isPluginActionsPermission(BasePermission):
+    @staticmethod
+    def has_object_permission(request, view, obj):
+        # only an admin or superuser can update or delete plugins
+        if request.user.has_membership():
+            return request.user.membership.is_admin
+        else:
+            return request.user.is_superuser

api_app/pivots_manager/serializers.py

@@ -3,7 +3,7 @@
 
 from api_app.analyzers_manager.models import AnalyzerConfig
 from api_app.connectors_manager.models import ConnectorConfig
-from api_app.models import Job, PythonModule
+from api_app.models import Job, PluginConfig, PythonModule
 from api_app.pivots_manager.models import PivotConfig, PivotMap, PivotReport
 from api_app.playbooks_manager.models import PlaybookConfig
 from api_app.serializers.plugin import (
@@ -77,7 +77,9 @@ class PivotConfigSerializer(PythonConfigSerializer):
     python_module = rfs.SlugRelatedField(
         queryset=PythonModule.objects.all(), slug_field="module"
     )
-    plugin_config = rfs.DictField(write_only=True, required=False)
+    plugin_config = rfs.ListField(
+        child=rfs.DictField(), write_only=True, required=False
+    )
 
     class Meta:
         model = PivotConfig
@@ -102,14 +104,33 @@ def create(self, validated_data):
         pc = super().create(validated_data)
 
         # create plugin config
-        if plugin_config:
+        for config in plugin_config:
             plugin_config_serializer = PluginConfigSerializer(
-                data=plugin_config, context={"request": self.context["request"]}
+                data=config, context={"request": self.context["request"]}
             )
             plugin_config_serializer.is_valid(raise_exception=True)
             plugin_config_serializer.save()
         return pc
 
+    def update(self, instance, validated_data):
+        plugin_config = validated_data.pop("plugin_config", [])
+        pc = super().update(instance, validated_data)
+
+        # update plugin config
+        for config in plugin_config:
+            plugin_config_serializer = PluginConfigSerializer(
+                data=config, context={"request": self.context["request"]}
+            )
+            plugin_config_serializer.is_valid(raise_exception=True)
+            PluginConfig.objects.filter(
+                owner=self.context["request"].user,
+                analyzer_config=plugin_config_serializer.validated_data[
+                    "analyzer_config"
+                ],
+                parameter=plugin_config_serializer.validated_data["parameter"],
+            ).update_or_create(plugin_config_serializer.validated_data)
+        return pc
+
 
 class PivotConfigSerializerForMigration(PythonConfigSerializerForMigration):
     related_analyzer_configs = rfs.SlugRelatedField(

api_app/serializers/plugin.py

@@ -272,7 +272,6 @@ class PythonConfigSerializer(AbstractConfigSerializer):
 
     class Meta:
         exclude = [
-            "python_module",
             "routing_key",
             "soft_time_limit",
             "health_check_status",