detect-it-easy analyzer, closes #1590

.gitignore

@@ -45,3 +45,6 @@ coverage.xml
 *.cover
 .hypothesis/
 /.env
+
+# post run dev
+integrations/malware_tools_analyzers/clamav/sigs

api_app/analyzers_manager/file_analyzers/detectiteasy.py

@@ -0,0 +1,60 @@
+import logging
+
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
+from tests.mock_utils import MockUpResponse
+
+logger = logging.getLogger(__name__)
+
+
+class DetectItEasy(FileAnalyzer, DockerBasedAnalyzer):
+    name: str = "executable_analyzer"
+    url: str = "http://malware_tools_analyzers:4002/die"
+    # http request polling max number of tries
+    max_tries: int = 10
+    # interval between http request polling (in secs)
+    poll_distance: int = 1
+
+    def update(self):
+        pass
+
+    def run(self):
+        fname = str(self.filename).replace("/", "_").replace(" ", "_")
+        # get the file to send
+        binary = self.read_file_bytes()
+        args = [f"@{fname}", "--json"]
+        req_data = {
+            "args": args,
+        }
+        req_files = {fname: binary}
+        logger.info(
+            f"Running {self.analyzer_name} on {self.filename} with args: {args}"
+        )
+        report = self._docker_run(req_data, req_files, analyzer_name=self.analyzer_name)
+        if not report:
+            self.report.errors.append("DIE does not support the file")
+            return {}
+        return report
+
+    @staticmethod
+    def mocked_docker_analyzer_get(*args, **kwargs):
+        return MockUpResponse(
+            {
+                "report": {
+                    "arch": "NOEXEC",
+                    "mode": "Unknown",
+                    "type": "Unknown",
+                    "detects": [
+                        {
+                            "name": "Zip",
+                            "type": "archive",
+                            "string": "archive: Zip(2.0)[38.5%,1 file]",
+                            "options": "38.5%,1 file",
+                            "version": "2.0",
+                        }
+                    ],
+                    "filetype": "Binary",
+                    "endianess": "LE",
+                }
+            },
+            200,
+        )

api_app/analyzers_manager/migrations/0094_analyzer_config_detectiteasy.py

@@ -0,0 +1,117 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+)
+
+plugin = {
+    "python_module": {
+        "health_check_schedule": None,
+        "update_schedule": None,
+        "module": "detectiteasy.DetectItEasy",
+        "base_path": "api_app.analyzers_manager.file_analyzers",
+    },
+    "name": "DetectItEasy",
+    "description": "[DetectItEasy](https://github.com/horsicq/Detect-It-Easy) is a program for determining types of files.",
+    "disabled": False,
+    "soft_time_limit": 10,
+    "routing_key": "default",
+    "health_check_status": True,
+    "type": "file",
+    "docker_based": True,
+    "maximum_tlp": "RED",
+    "observable_supported": [],
+    "supported_filetypes": [],
+    "run_hash": False,
+    "run_hash_type": "",
+    "not_supported_filetypes": [],
+    "model": "analyzers_manager.AnalyzerConfig",
+}
+
+params = []
+
+values = []
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0062_alter_parameter_python_module"),
+        ("analyzers_manager", "0093_analyzer_config_ailtyposquatting"),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

docs/source/Usage.md

@@ -100,6 +100,7 @@ The following is the list of the available analyzers you can run out-of-the-box.
 * `Suricata`: Analyze PCAPs with open IDS signatures with [Suricata engine](https://github.com/OISF/suricata)
 * `Thug_HTML_Info`: Perform hybrid dynamic/static analysis on a HTML file using [Thug low-interaction honeyclient](https://thug-honeyclient.readthedocs.io/)
 * `Xlm_Macro_Deobfuscator`: [XlmMacroDeobfuscator](https://github.com/DissectMalware/XLMMacroDeobfuscator) deobfuscate xlm macros
+* `DetectItEasy`:[DetectItEasy](https://github.com/horsicq/Detect-It-Easy) is a program for determining types of files.
 * `Yara`: scan a file with
   * [ATM malware yara rules](https://github.com/fboldewin/YARA-rules)
   * [bartblaze yara rules](https://github.com/bartblaze/Yara-rules)

integrations/malware_tools_analyzers/Dockerfile

@@ -97,6 +97,12 @@ RUN python3 -m venv venv \
     && mkdir -p /tmp/thug/logs \
     && chown -R ${USER}:${USER} /tmp/thug/logs
 
+WORKDIR ${PROJECT_PATH}/die
+RUN apt-get install --no-install-recommends -y wget tar libglib2.0-0 && \
+    wget -q https://github.com/horsicq/DIE-engine/releases/download/3.01/die_lin64_portable_3.01.tar.gz && \
+    tar -xzf die_lin64_portable_3.01.tar.gz
+
+
 # prepare fangfrisch installation
 COPY crontab /etc/cron.d/crontab
 RUN mkdir -m 0770 -p /var/lib/fangfrisch \

integrations/malware_tools_analyzers/app.py

@@ -175,6 +175,12 @@ def intercept_thug_result(context, future: Future) -> None:
     command_name="/opt/deploy/qiling/venv/bin/python3 /opt/deploy/qiling/analyze.py",
 )
 
+# diec is the command for Detect It Easy
+shell2http.register_command(
+    endpoint="die",
+    command_name="/opt/deploy/die/die_lin64_portable/base/diec",
+)
+
 # with this, we can make http calls to the endpoint: /thug
 shell2http.register_command(
     endpoint="thug",

integrations/malware_tools_analyzers/entrypoint.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# diec analyzer variable
+export LD_LIBRARY_PATH="/opt/deploy/die/die_lin64_portable/base:$LD_LIBRARY_PATH"
 # without this makedirs the Dockerfile is not able to create new directories in volumes that already exist
 mkdir -p /var/run/clamav ${LOG_PATH} ${LOG_PATH}/clamav
 chown -R clamav:${USER} /var/lib/clamav /var/run/clamav ${LOG_PATH}