Engine

api_app/admin.py

@@ -49,10 +49,8 @@ class JobAdminView(CustomAdminView):
         "id",
         "status",
         "user",
-        "observable_name",
-        "observable_classification",
-        "file_name",
-        "file_mimetype",
+        "get_analyzable_name",
+        "get_analyzable_classification",
         "received_request_time",
         "analyzers_executed",
         "connectors_executed",
@@ -64,13 +62,16 @@ class JobAdminView(CustomAdminView):
         "user",
         "status",
     )
-    search_fields = (
-        "md5",
-        "observable_name",
-        "file_name",
-    )
     list_filter = ("status", "user", "tags")
 
+    @admin.display(description="Name")
+    def get_analyzable_name(self, instance):
+        return instance.analyzable.name
+
+    @admin.display(description="Classification")
+    def get_analyzable_classification(self, instance):
+        return instance.analyzable.classification
+
     @staticmethod
     def has_add_permission(request: HttpRequest) -> bool:
         return False

api_app/analyzables_manager/admin.py

@@ -0,0 +1,11 @@
+from django.contrib import admin
+
+from api_app.analyzables_manager.models import Analyzable
+
+
+@admin.register(Analyzable)
+class AnalyzableAdmin(admin.ModelAdmin):
+    list_display = ["pk", "name", "sha1", "sha256", "md5"]
+    search_fields = ["name", "sha1", "sha256", "md5"]
+    ordering = ["name"]
+    list_filter = ["discovery_date"]

api_app/analyzables_manager/migrations/0001_initial.py

@@ -0,0 +1,70 @@
+# Generated by Django 4.2.17 on 2025-01-22 08:59
+
+from django.db import migrations, models
+from django.utils.timezone import now
+
+import api_app.defaults
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Analyzable",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("md5", models.CharField(max_length=255, unique=True, editable=False)),
+                (
+                    "sha256",
+                    models.CharField(max_length=255, unique=True, editable=False),
+                ),
+                ("sha1", models.CharField(max_length=255, unique=True, editable=False)),
+                ("name", models.CharField(max_length=255)),
+                (
+                    "mimetype",
+                    models.CharField(
+                        blank=True, max_length=80, null=True, default=None
+                    ),
+                ),
+                (
+                    "file",
+                    models.FileField(
+                        null=True,
+                        default=None,
+                        blank=True,
+                        upload_to=api_app.defaults.file_directory_path,
+                    ),
+                ),
+                (
+                    "classification",
+                    models.CharField(
+                        max_length=100,
+                        choices=[
+                            ("ip", "Ip"),
+                            ("url", "Url"),
+                            ("domain", "Domain"),
+                            ("hash", "Hash"),
+                            ("generic", "Generic"),
+                            ("file", "File"),
+                        ],
+                    ),
+                ),
+                ("discovery_date", models.DateTimeField(default=now)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

api_app/analyzables_manager/migrations/0002_migrate_data.py

@@ -0,0 +1,55 @@
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    Job = apps.get_model("api_app", "Job")
+    Analyzable = apps.get_model("analyzables_manager", "Analyzable")
+    for job in Job.objects.all().order_by("received_request_time"):
+        if job.is_sample:
+            obj, created = Analyzable.objects.get_or_create(
+                md5=job.md5,
+                defaults={
+                    "file": job.file,
+                    "mimetype": job.file_mimetype,
+                    "name": job.file_name,
+                    "md5": job.md5,
+                    "classification": "sample",
+                    "discovery_date": job.received_request_time,
+                },
+            )
+            if created:
+                p = job.file.path
+                try:
+                    p.rename(p.parent / job.md5)
+                except Exception:
+                    ...
+                else:
+                    job.file.name = job.md5
+        else:
+            obj, created = Analyzable.objects.get_or_create(
+                md5=job.md5,
+                defaults={
+                    "name": job.observable_name,
+                    "md5": job.md5,
+                    "classification": job.observable_classification,
+                    "discovery_date": job.received_request_time,
+                },
+            )
+        if created:
+            obj.full_clean()
+            obj.save()
+        job.analyzable = obj
+        job.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("api_app", "0067_add_analyzable"),
+        ("analyzables_manager", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate, migrations.RunPython.noop),
+    ]

api_app/analyzables_manager/migrations/0003_analyzable_analyzables_classif_adf7ca_idx_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.17 on 2025-01-23 14:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzables_manager", "0002_migrate_data"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="analyzable",
+            index=models.Index(
+                fields=["classification"], name="analyzables_classif_adf7ca_idx"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="analyzable",
+            index=models.Index(
+                fields=["mimetype"], name="analyzables_mimetyp_321d7d_idx"
+            ),
+        ),
+    ]

api_app/analyzables_manager/models.py

@@ -0,0 +1,92 @@
+from typing import Type, Union
+
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils.timezone import now
+
+from api_app.analyzables_manager.queryset import AnalyzableQuerySet
+from api_app.choices import Classification
+from api_app.data_model_manager.models import (
+    BaseDataModel,
+    DomainDataModel,
+    FileDataModel,
+    IPDataModel,
+)
+from api_app.defaults import file_directory_path
+from api_app.helpers import calculate_md5, calculate_sha1, calculate_sha256
+
+
+class Analyzable(models.Model):
+    name = models.CharField(max_length=255)
+    discovery_date = models.DateTimeField(default=now)
+    md5 = models.CharField(max_length=255, unique=True, editable=False)
+    sha256 = models.CharField(max_length=255, unique=True, editable=False)
+    sha1 = models.CharField(max_length=255, unique=True, editable=False)
+    classification = models.CharField(max_length=100, choices=Classification.choices)
+    mimetype = models.CharField(max_length=80, blank=True, null=True, default=None)
+    file = models.FileField(
+        upload_to=file_directory_path, null=True, blank=True, default=None
+    )
+
+    objects = AnalyzableQuerySet.as_manager()
+
+    class Meta:
+        indexes = [
+            models.Index(fields=["classification"]),
+            models.Index(fields=["mimetype"]),
+        ]
+
+    def __str__(self):
+        return self.name
+
+    @property
+    def analyzed_object(self):
+        return self.file if self.is_sample else self.name
+
+    @property
+    def is_sample(self) -> bool:
+        return self.classification == Classification.FILE.value
+
+    def get_data_model_class(self) -> Type[BaseDataModel]:
+        if self.classification == Classification.IP.value:
+            return IPDataModel
+        elif self.classification in [
+            Classification.URL.value,
+            Classification.DOMAIN.value,
+        ]:
+            return DomainDataModel
+        elif self.classification in [
+            Classification.HASH.value,
+            Classification.FILE.value,
+        ]:
+            return FileDataModel
+        else:
+            raise NotImplementedError()
+
+    def _set_hashes(self, value: Union[str, bytes]):
+        if isinstance(value, str):
+            value = value.encode("utf-8")
+        if not self.md5:
+            self.md5 = calculate_md5(value)
+        if not self.sha256:
+            self.sha256 = calculate_sha256(value)
+        if not self.sha1:
+            self.sha1 = calculate_sha1(value)
+
+    def clean(self):
+        if self.classification == Classification.FILE.value:
+            if not self.mimetype or not self.file:
+                raise ValidationError("Mimetype and file must be set for samples")
+            content = self.read()
+        else:
+            if self.mimetype or self.file:
+                raise ValidationError(
+                    "Mimetype and file must not be set for observables"
+                )
+            content = self.name
+        self._set_hashes(content)
+
+    def read(self) -> bytes:
+        if self.classification == Classification.FILE.value:
+            self.file.seek(0)
+            return self.file.read()

api_app/analyzables_manager/queryset.py

@@ -0,0 +1,22 @@
+from django.db.models import QuerySet
+
+
+class AnalyzableQuerySet(QuerySet):
+
+    def visible_for_user(self, user):
+        from api_app.models import Job
+
+        jobs = (
+            Job.objects.visible_for_user(user)
+            .values("analyzable")
+            .distinct()
+            .values_list("pk", flat=True)
+        )
+        return self.filter(pk__in=jobs)
+
+    def create(self, *args, **kwargs):
+        obj = self.model(**kwargs)
+        self._for_write = True
+        obj.full_clean()
+        obj.save(force_insert=True, using=self.db)
+        return obj

api_app/analyzables_manager/serializers.py

@@ -0,0 +1,12 @@
+from rest_framework.serializers import ModelSerializer
+
+from api_app.analyzables_manager.models import Analyzable
+from api_app.serializers.job import JobRelatedField
+
+
+class AnalyzableSerializer(ModelSerializer):
+    jobs = JobRelatedField(many=True, read_only=True)
+
+    class Meta:
+        model = Analyzable
+        fields = "__all__"

api_app/analyzables_manager/signals.py

@@ -0,0 +1,19 @@
+from django.db import models
+from django.dispatch import receiver
+
+from api_app.analyzables_manager.models import Analyzable
+
+
+@receiver(models.signals.pre_delete, sender=Analyzable)
+def pre_delete_analyzable(sender, instance: Analyzable, **kwargs):
+    """
+    Signal receiver for the pre_delete signal of the Analyzable model.
+    Deletes the associated file if it exists.
+
+    Args:
+        sender (Model): The model class sending the signal.
+        instance (Analyzable): The instance of the model being deleted.
+        **kwargs: Additional keyword arguments.
+    """
+    if instance.file:
+        instance.file.delete()

api_app/analyzables_manager/urls.py

@@ -0,0 +1,18 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+from django.urls import include, path
+from rest_framework import routers
+
+from api_app.analyzables_manager.views import AnalyzableViewSet
+
+# Routers provide an easy way of automatically determining the URL conf.
+
+
+router = routers.DefaultRouter(trailing_slash=False)
+router.register(r"analyzable", AnalyzableViewSet, basename="analyzable")
+
+urlpatterns = [
+    # Viewsets
+    path(r"", include(router.urls)),
+]

api_app/analyzables_manager/views.py

@@ -0,0 +1,14 @@
+from rest_framework import viewsets
+from rest_framework.permissions import IsAuthenticated
+
+from api_app.analyzables_manager.serializers import AnalyzableSerializer
+
+
+class AnalyzableViewSet(viewsets.ReadOnlyModelViewSet):
+
+    serializer_class = AnalyzableSerializer
+    permission_classes = [IsAuthenticated]
+
+    def get_queryset(self):
+        user = self.request.user
+        return super().get_queryset().visible_for_user(user)