Skip to content
This repository was archived by the owner on Nov 2, 2024. It is now read-only.

Commit 85f26a6

Browse files
g4zeg4zemlodic
authored and
Michalsus
committed
detect-it-easy analyzer, closes intelowlproject#1590 (intelowlproject#2354)
* die * tweeks * codefactor * codefactor * ypo * gitignore * typo fix * detectiteasyyyyy * tests * supported files * msdos * logs, file support, soft t/o, poll * migrate * for all files * docker_based_true * params * tests debug[1] * Update api_app/analyzers_manager/migrations/0094_analyzer_config_detectiteasy.py * Update api_app/analyzers_manager/file_analyzers/detectiteasy.py --------- Co-authored-by: g4ze <[email protected]> Co-authored-by: Matteo Lodi <[email protected]>
1 parent a1613ac commit 85f26a6

File tree

8 files changed

+264
-1
lines changed

8 files changed

+264
-1
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,6 @@ coverage.xml
4545
*.cover
4646
.hypothesis/
4747
/.env
48+
49+
# post run dev
50+
integrations/malware_tools_analyzers/clamav/sigs

‎api_app/analyzers_manager/file_analyzers/detectiteasy.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import logging
2+
3+
from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
4+
from tests.mock_utils import MockUpResponse
5+
6+
logger = logging.getLogger(__name__)
7+
8+
9+
class DetectItEasy(FileAnalyzer, DockerBasedAnalyzer):
10+
name: str = "executable_analyzer"
11+
url: str = "http://malware_tools_analyzers:4002/die"
12+
# http request polling max number of tries
13+
max_tries: int = 10
14+
# interval between http request polling (in secs)
15+
poll_distance: int = 1
16+
17+
def update(self):
18+
pass
19+
20+
def run(self):
21+
fname = str(self.filename).replace("/", "_").replace(" ", "_")
22+
# get the file to send
23+
binary = self.read_file_bytes()
24+
args = [f"@{fname}", "--json"]
25+
req_data = {
26+
"args": args,
27+
}
28+
req_files = {fname: binary}
29+
logger.info(
30+
f"Running {self.analyzer_name} on {self.filename} with args: {args}"
31+
)
32+
report = self._docker_run(req_data, req_files, analyzer_name=self.analyzer_name)
33+
if not report:
34+
self.report.errors.append("DIE did not detect the file type")
35+
return {}
36+
return report
37+
38+
@staticmethod
39+
def mocked_docker_analyzer_get(*args, **kwargs):
40+
return MockUpResponse(
41+
{
42+
"report": {
43+
"arch": "NOEXEC",
44+
"mode": "Unknown",
45+
"type": "Unknown",
46+
"detects": [
47+
{
48+
"name": "Zip",
49+
"type": "archive",
50+
"string": "archive: Zip(2.0)[38.5%,1 file]",
51+
"options": "38.5%,1 file",
52+
"version": "2.0",
53+
}
54+
],
55+
"filetype": "Binary",
56+
"endianess": "LE",
57+
}
58+
},
59+
200,
60+
)

‎api_app/analyzers_manager/migrations/0094_analyzer_config_detectiteasy.py‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
from django.db import migrations
2+
from django.db.models.fields.related_descriptors import (
3+
ForwardManyToOneDescriptor,
4+
ForwardOneToOneDescriptor,
5+
ManyToManyDescriptor,
6+
)
7+
8+
plugin = {
9+
"python_module": {
10+
"health_check_schedule": None,
11+
"update_schedule": None,
12+
"module": "detectiteasy.DetectItEasy",
13+
"base_path": "api_app.analyzers_manager.file_analyzers",
14+
},
15+
"name": "DetectItEasy",
16+
"description": "[DetectItEasy](https://github.com/horsicq/Detect-It-Easy) is a program for determining types of files.",
17+
"disabled": False,
18+
"soft_time_limit": 10,
19+
"routing_key": "default",
20+
"health_check_status": True,
21+
"type": "file",
22+
"docker_based": True,
23+
"maximum_tlp": "RED",
24+
"observable_supported": [],
25+
"supported_filetypes": [],
26+
"run_hash": False,
27+
"run_hash_type": "",
28+
"not_supported_filetypes": [],
29+
"model": "analyzers_manager.AnalyzerConfig",
30+
}
31+
32+
params = [
33+
{
34+
"python_module": {
35+
"module": "detectiteasy.DetectItEasy",
36+
"base_path": "api_app.analyzers_manager.file_analyzers",
37+
},
38+
"name": "max_tries",
39+
"type": "int",
40+
"description": "max_tries for detect it easy",
41+
"is_secret": False,
42+
"required": False,
43+
},
44+
{
45+
"python_module": {
46+
"module": "detectiteasy.DetectItEasy",
47+
"base_path": "api_app.analyzers_manager.file_analyzers",
48+
},
49+
"name": "poll_distance",
50+
"type": "int",
51+
"description": "poll_distance for detect it easy",
52+
"is_secret": False,
53+
"required": False,
54+
},
55+
]
56+
57+
values = [
58+
{
59+
"parameter": {
60+
"python_module": {
61+
"module": "detectiteasy.DetectItEasy",
62+
"base_path": "api_app.analyzers_manager.file_analyzers",
63+
},
64+
"name": "max_tries",
65+
"type": "int",
66+
"description": "max_tries for detect it easy",
67+
"is_secret": False,
68+
"required": False,
69+
},
70+
"analyzer_config": "DetectItEasy",
71+
"connector_config": None,
72+
"visualizer_config": None,
73+
"ingestor_config": None,
74+
"pivot_config": None,
75+
"for_organization": False,
76+
"value": 10,
77+
"updated_at": "2024-06-05T10:38:28.119622Z",
78+
"owner": None,
79+
},
80+
{
81+
"parameter": {
82+
"python_module": {
83+
"module": "detectiteasy.DetectItEasy",
84+
"base_path": "api_app.analyzers_manager.file_analyzers",
85+
},
86+
"name": "poll_distance",
87+
"type": "int",
88+
"description": "poll_distance for detect it easy",
89+
"is_secret": False,
90+
"required": False,
91+
},
92+
"analyzer_config": "DetectItEasy",
93+
"connector_config": None,
94+
"visualizer_config": None,
95+
"ingestor_config": None,
96+
"pivot_config": None,
97+
"for_organization": False,
98+
"value": 1,
99+
"updated_at": "2024-06-05T10:38:28.426691Z",
100+
"owner": None,
101+
},
102+
]
103+
104+
105+
def _get_real_obj(Model, field, value):
106+
def _get_obj(Model, other_model, value):
107+
if isinstance(value, dict):
108+
real_vals = {}
109+
for key, real_val in value.items():
110+
real_vals[key] = _get_real_obj(other_model, key, real_val)
111+
value = other_model.objects.get_or_create(**real_vals)[0]
112+
# it is just the primary key serialized
113+
else:
114+
if isinstance(value, int):
115+
if Model.__name__ == "PluginConfig":
116+
value = other_model.objects.get(name=plugin["name"])
117+
else:
118+
value = other_model.objects.get(pk=value)
119+
else:
120+
value = other_model.objects.get(name=value)
121+
return value
122+
123+
if (
124+
type(getattr(Model, field))
125+
in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
126+
and value
127+
):
128+
other_model = getattr(Model, field).get_queryset().model
129+
value = _get_obj(Model, other_model, value)
130+
elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
131+
other_model = getattr(Model, field).rel.model
132+
value = [_get_obj(Model, other_model, val) for val in value]
133+
return value
134+
135+
136+
def _create_object(Model, data):
137+
mtm, no_mtm = {}, {}
138+
for field, value in data.items():
139+
value = _get_real_obj(Model, field, value)
140+
if type(getattr(Model, field)) is ManyToManyDescriptor:
141+
mtm[field] = value
142+
else:
143+
no_mtm[field] = value
144+
try:
145+
o = Model.objects.get(**no_mtm)
146+
except Model.DoesNotExist:
147+
o = Model(**no_mtm)
148+
o.full_clean()
149+
o.save()
150+
for field, value in mtm.items():
151+
attribute = getattr(o, field)
152+
if value is not None:
153+
attribute.set(value)
154+
return False
155+
return True
156+
157+
158+
def migrate(apps, schema_editor):
159+
Parameter = apps.get_model("api_app", "Parameter")
160+
PluginConfig = apps.get_model("api_app", "PluginConfig")
161+
python_path = plugin.pop("model")
162+
Model = apps.get_model(*python_path.split("."))
163+
if not Model.objects.filter(name=plugin["name"]).exists():
164+
exists = _create_object(Model, plugin)
165+
if not exists:
166+
for param in params:
167+
_create_object(Parameter, param)
168+
for value in values:
169+
_create_object(PluginConfig, value)
170+
171+
172+
def reverse_migrate(apps, schema_editor):
173+
python_path = plugin.pop("model")
174+
Model = apps.get_model(*python_path.split("."))
175+
Model.objects.get(name=plugin["name"]).delete()
176+
177+
178+
class Migration(migrations.Migration):
179+
atomic = False
180+
dependencies = [
181+
("api_app", "0062_alter_parameter_python_module"),
182+
("analyzers_manager", "0093_analyzer_config_ailtyposquatting"),
183+
]
184+
185+
operations = [migrations.RunPython(migrate, reverse_migrate)]

‎docs/source/Usage.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ The following is the list of the available analyzers you can run out-of-the-box.
100100
* `Suricata`: Analyze PCAPs with open IDS signatures with [Suricata engine](https://github.com/OISF/suricata)
101101
* `Thug_HTML_Info`: Perform hybrid dynamic/static analysis on a HTML file using [Thug low-interaction honeyclient](https://thug-honeyclient.readthedocs.io/)
102102
* `Xlm_Macro_Deobfuscator`: [XlmMacroDeobfuscator](https://github.com/DissectMalware/XLMMacroDeobfuscator) deobfuscate xlm macros
103+
* `DetectItEasy`:[DetectItEasy](https://github.com/horsicq/Detect-It-Easy) is a program for determining types of files.
103104
* `Yara`: scan a file with
104105
* [ATM malware yara rules](https://github.com/fboldewin/YARA-rules)
105106
* [bartblaze yara rules](https://github.com/bartblaze/Yara-rules)

‎integrations/malware_tools_analyzers/Dockerfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,12 @@ RUN python3 -m venv venv \
9797
&& mkdir -p /tmp/thug/logs \
9898
&& chown -R ${USER}:${USER} /tmp/thug/logs
9999

100+
WORKDIR ${PROJECT_PATH}/die
101+
RUN apt-get install --no-install-recommends -y wget tar libglib2.0-0 && \
102+
wget -q https://github.com/horsicq/DIE-engine/releases/download/3.01/die_lin64_portable_3.01.tar.gz && \
103+
tar -xzf die_lin64_portable_3.01.tar.gz
104+
105+
100106
# prepare fangfrisch installation
101107
COPY crontab /etc/cron.d/crontab
102108
RUN mkdir -m 0770 -p /var/lib/fangfrisch \

‎integrations/malware_tools_analyzers/app.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,12 @@ def intercept_thug_result(context, future: Future) -> None:
175175
command_name="/opt/deploy/qiling/venv/bin/python3 /opt/deploy/qiling/analyze.py",
176176
)
177177

178+
# diec is the command for Detect It Easy
179+
shell2http.register_command(
180+
endpoint="die",
181+
command_name="/opt/deploy/die/die_lin64_portable/base/diec",
182+
)
183+
178184
# with this, we can make http calls to the endpoint: /thug
179185
shell2http.register_command(
180186
endpoint="thug",

‎integrations/malware_tools_analyzers/entrypoint.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#!/bin/bash
2+
# diec analyzer variable
3+
export LD_LIBRARY_PATH="/opt/deploy/die/die_lin64_portable/base:$LD_LIBRARY_PATH"
24
# without this makedirs the Dockerfile is not able to create new directories in volumes that already exist
35
mkdir -p /var/run/clamav ${LOG_PATH} ${LOG_PATH}/clamav
46
chown -R clamav:${USER} /var/lib/clamav /var/run/clamav ${LOG_PATH}

‎tests/api_app/test_api.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ def test_analyze_file__pcap(self):
129129
self.assertEqual(md5, job.md5)
130130

131131
self.assertCountEqual(
132-
["Suricata", "YARAify_File_Scan", "Hfinger"],
132+
["Suricata", "YARAify_File_Scan", "Hfinger", "DetectItEasy"],
133133
list(job.analyzers_to_execute.all().values_list("name", flat=True)),
134134
)
135135

0 commit comments

Comments
 (0)