Skip to content

Commit ecf9bb9

Browse files
g4zeg4ze
committed
Merge branch 'develop' of https://github.com/intelowlproject/IntelOwl into Polyswarm-analyzer-closes#1255
2 parents eecb6e9 + a508531 commit ecf9bb9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1697
-366
lines changed

‎api_app/admin.py‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
22
# See the file 'LICENSE' for copying permission.
3+
from gettext import ngettext
34
from typing import Any
45

5-
from django.contrib import admin
6+
from django.contrib import admin, messages
67
from django.contrib.admin import widgets
78
from django.db.models import JSONField, ManyToManyField
89
from django.http import HttpRequest
@@ -69,10 +70,12 @@ class JobAdminView(CustomAdminView):
6970
)
7071
list_filter = ("status", "user", "tags")
7172

72-
def has_add_permission(self, request: HttpRequest) -> bool:
73+
@staticmethod
74+
def has_add_permission(request: HttpRequest) -> bool:
7375
return False
7476

75-
def has_change_permission(self, request: HttpRequest, obj=None) -> bool:
77+
@staticmethod
78+
def has_change_permission(request: HttpRequest, obj=None) -> bool:
7679
return False
7780

7881
@admin.display(description="Tags")
@@ -151,7 +154,8 @@ class AbstractReportAdminView(CustomAdminView):
151154
def has_add_permission(request):
152155
return False
153156

154-
def has_change_permission(self, request: HttpRequest, obj=None) -> bool:
157+
@staticmethod
158+
def has_change_permission(request: HttpRequest, obj=None) -> bool:
155159
return False
156160

157161

@@ -193,6 +197,7 @@ class AbstractConfigAdminView(CustomAdminView):
193197
list_filter = ("disabled",)
194198
# allow to clone the object
195199
save_as = True
200+
actions = ["disable", "enable"]
196201

197202
@admin.display(description="Disabled in orgs")
198203
def disabled_in_orgs(self, instance: AbstractConfig):
@@ -202,6 +207,34 @@ def disabled_in_orgs(self, instance: AbstractConfig):
202207
)
203208
)
204209

210+
def disable(self, request, queryset):
211+
counter = queryset.update(disabled=True)
212+
self.message_user(
213+
request,
214+
ngettext(
215+
f"{counter} {queryset.model._meta.verbose_name} was disabled.",
216+
f"{counter} {queryset.model._meta.verbose_name_plural} were disabled.",
217+
counter,
218+
),
219+
messages.SUCCESS,
220+
)
221+
222+
disable.short_description = "Disable configurations"
223+
224+
def enable(self, request, queryset):
225+
counter = queryset.update(disabled=False)
226+
self.message_user(
227+
request,
228+
ngettext(
229+
f"{counter} {queryset.model._meta.verbose_name} was enabled.",
230+
f"{counter} {queryset.model._meta.verbose_name_plural} were enabled.",
231+
counter,
232+
),
233+
messages.SUCCESS,
234+
)
235+
236+
enable.short_description = "Enable configurations"
237+
205238

206239
class PythonConfigAdminView(AbstractConfigAdminView):
207240
list_display = AbstractConfigAdminView.list_display + ("routing_key",)

‎api_app/analyzers_manager/migrations/0112_analyzer_config_criminalip_scan.py‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
from django.db import migrations
2+
from django.db.models.fields.related_descriptors import (
3+
ForwardManyToOneDescriptor,
4+
ForwardOneToOneDescriptor,
5+
ManyToManyDescriptor,
6+
)
7+
8+
plugin = {
9+
"python_module": {
10+
"health_check_schedule": None,
11+
"update_schedule": None,
12+
"module": "criminalip.criminalip_scan.CriminalIpScan",
13+
"base_path": "api_app.analyzers_manager.observable_analyzers",
14+
},
15+
"name": "CriminalIpScan",
16+
"description": "[Criminal IP](https://www.criminalip.io/) is an OSINT search engine specialized in attack surface assessment and threat hunting. It offers extensive cyber threat intelligence, including device reputation, geolocation, IP reputation for C2 or scanners, domain safety, malicious link detection, and APT attack vectors via search and API.",
17+
"disabled": False,
18+
"soft_time_limit": 10,
19+
"routing_key": "default",
20+
"health_check_status": True,
21+
"type": "observable",
22+
"docker_based": False,
23+
"maximum_tlp": "AMBER",
24+
"observable_supported": ["ip", "domain", "generic"],
25+
"supported_filetypes": [],
26+
"run_hash": False,
27+
"run_hash_type": "",
28+
"not_supported_filetypes": [],
29+
"model": "analyzers_manager.AnalyzerConfig",
30+
}
31+
32+
params = [
33+
{
34+
"python_module": {
35+
"module": "criminalip.criminalip_scan.CriminalIpScan",
36+
"base_path": "api_app.analyzers_manager.observable_analyzers",
37+
},
38+
"name": "api_key",
39+
"type": "str",
40+
"description": "api key for criminal ip",
41+
"is_secret": True,
42+
"required": False,
43+
},
44+
{
45+
"python_module": {
46+
"module": "criminalip.criminalip_scan.CriminalIpScan",
47+
"base_path": "api_app.analyzers_manager.observable_analyzers",
48+
},
49+
"name": "malicious_info",
50+
"type": "bool",
51+
"description": "for IP, default endpoint",
52+
"is_secret": False,
53+
"required": False,
54+
},
55+
{
56+
"python_module": {
57+
"module": "criminalip.criminalip_scan.CriminalIpScan",
58+
"base_path": "api_app.analyzers_manager.observable_analyzers",
59+
},
60+
"name": "privacy_threat",
61+
"type": "bool",
62+
"description": "for IP, privacy-threat endpoint",
63+
"is_secret": False,
64+
"required": False,
65+
},
66+
{
67+
"python_module": {
68+
"module": "criminalip.criminalip_scan.CriminalIpScan",
69+
"base_path": "api_app.analyzers_manager.observable_analyzers",
70+
},
71+
"name": "is_safe_dns_server",
72+
"type": "bool",
73+
"description": "for IP, is-safe-dns-server endpoint",
74+
"is_secret": False,
75+
"required": False,
76+
},
77+
{
78+
"python_module": {
79+
"module": "criminalip.criminalip_scan.CriminalIpScan",
80+
"base_path": "api_app.analyzers_manager.observable_analyzers",
81+
},
82+
"name": "suspicious_info",
83+
"type": "bool",
84+
"description": "for IP, suspicious_info endpoint",
85+
"is_secret": False,
86+
"required": False,
87+
},
88+
{
89+
"python_module": {
90+
"module": "criminalip.criminalip_scan.CriminalIpScan",
91+
"base_path": "api_app.analyzers_manager.observable_analyzers",
92+
},
93+
"name": "banner_search",
94+
"type": "bool",
95+
"description": "for generics",
96+
"is_secret": False,
97+
"required": False,
98+
},
99+
{
100+
"python_module": {
101+
"module": "criminalip.criminalip_scan.CriminalIpScan",
102+
"base_path": "api_app.analyzers_manager.observable_analyzers",
103+
},
104+
"name": "banner_stats",
105+
"type": "bool",
106+
"description": "for generics",
107+
"is_secret": False,
108+
"required": False,
109+
},
110+
]
111+
112+
values = []
113+
114+
115+
def _get_real_obj(Model, field, value):
116+
def _get_obj(Model, other_model, value):
117+
if isinstance(value, dict):
118+
real_vals = {}
119+
for key, real_val in value.items():
120+
real_vals[key] = _get_real_obj(other_model, key, real_val)
121+
value = other_model.objects.get_or_create(**real_vals)[0]
122+
# it is just the primary key serialized
123+
else:
124+
if isinstance(value, int):
125+
if Model.__name__ == "PluginConfig":
126+
value = other_model.objects.get(name=plugin["name"])
127+
else:
128+
value = other_model.objects.get(pk=value)
129+
else:
130+
value = other_model.objects.get(name=value)
131+
return value
132+
133+
if (
134+
type(getattr(Model, field))
135+
in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
136+
and value
137+
):
138+
other_model = getattr(Model, field).get_queryset().model
139+
value = _get_obj(Model, other_model, value)
140+
elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
141+
other_model = getattr(Model, field).rel.model
142+
value = [_get_obj(Model, other_model, val) for val in value]
143+
return value
144+
145+
146+
def _create_object(Model, data):
147+
mtm, no_mtm = {}, {}
148+
for field, value in data.items():
149+
value = _get_real_obj(Model, field, value)
150+
if type(getattr(Model, field)) is ManyToManyDescriptor:
151+
mtm[field] = value
152+
else:
153+
no_mtm[field] = value
154+
try:
155+
o = Model.objects.get(**no_mtm)
156+
except Model.DoesNotExist:
157+
o = Model(**no_mtm)
158+
o.full_clean()
159+
o.save()
160+
for field, value in mtm.items():
161+
attribute = getattr(o, field)
162+
if value is not None:
163+
attribute.set(value)
164+
return False
165+
return True
166+
167+
168+
def migrate(apps, schema_editor):
169+
Parameter = apps.get_model("api_app", "Parameter")
170+
PluginConfig = apps.get_model("api_app", "PluginConfig")
171+
python_path = plugin.pop("model")
172+
Model = apps.get_model(*python_path.split("."))
173+
if not Model.objects.filter(name=plugin["name"]).exists():
174+
exists = _create_object(Model, plugin)
175+
if not exists:
176+
for param in params:
177+
_create_object(Parameter, param)
178+
for value in values:
179+
_create_object(PluginConfig, value)
180+
181+
PythonModule = apps.get_model("api_app", "PythonModule")
182+
# we will update the python module path
183+
pm = PythonModule.objects.get(
184+
module="criminalip.CriminalIp",
185+
base_path="api_app.analyzers_manager.observable_analyzers",
186+
)
187+
pm.module = "criminalip.criminalip.CriminalIp"
188+
pm.save()
189+
Model.objects.filter(name="CriminalIp").update(python_module=pm)
190+
191+
192+
def reverse_migrate(apps, schema_editor):
193+
python_path = plugin.pop("model")
194+
Model = apps.get_model(*python_path.split("."))
195+
Model.objects.get(name=plugin["name"]).delete()
196+
PythonModule = apps.get_model("api_app", "PythonModule")
197+
pm = PythonModule.objects.get(
198+
module="criminalip.criminalip.CriminalIp",
199+
base_path="api_app.analyzers_manager.observable_analyzers",
200+
)
201+
pm.module = "criminalip.CriminalIp"
202+
pm.save()
203+
Model.objects.filter(name="CriminalIp").update(python_module=pm)
204+
205+
206+
class Migration(migrations.Migration):
207+
atomic = False
208+
dependencies = [
209+
("api_app", "0062_alter_parameter_python_module"),
210+
("analyzers_manager", "0111_analyzer_config_criminalip"),
211+
]
212+
213+
operations = [migrations.RunPython(migrate, reverse_migrate)]

‎api_app/analyzers_manager/observable_analyzers/criminalip/__init__.py‎

Whitespace-only changes.

api_app/analyzers_manager/observable_analyzers/criminalip.py renamed to api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1+
import logging
12
from typing import Dict
23

34
import requests
45

56
from api_app.analyzers_manager import classes
67
from tests.mock_utils import MockUpResponse, if_mock_connections, patch
78

9+
from .criminalip_base import CriminalIpBase
810

9-
class CriminalIp(classes.ObservableAnalyzer):
10-
url = "https://api.criminalip.io"
11-
_api_key: str = None
11+
logger = logging.getLogger(__name__)
1212

13+
14+
class CriminalIp(classes.ObservableAnalyzer, CriminalIpBase):
1315
malicious_info: bool = True # IP
1416
privacy_threat: bool = False
1517
is_safe_dns_server: bool = False
@@ -18,14 +20,12 @@ class CriminalIp(classes.ObservableAnalyzer):
1820
banner_stats: bool = False
1921
hash_view: bool = True # domain
2022

21-
def update(self):
22-
pass
23-
2423
def make_request(self, url: str, params: Dict[str, str] = None) -> Dict:
25-
headers = {"x-api-key": self._api_key}
26-
resp = requests.get(url, headers=headers, params=params)
24+
resp = requests.get(url, headers=self.getHeaders(), params=params)
2725
resp.raise_for_status()
28-
return resp.json()
26+
resp = resp.json()
27+
logger.info(f"response from CriminalIp for {self.observable_name} -> {resp}")
28+
return resp
2929

3030
def run(self):
3131
URLs = {

‎api_app/analyzers_manager/observable_analyzers/criminalip/criminalip_base.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import abc
2+
3+
from api_app.analyzers_manager.classes import BaseAnalyzerMixin
4+
5+
6+
class CriminalIpBase(BaseAnalyzerMixin, metaclass=abc.ABCMeta):
7+
url = "https://api.criminalip.io"
8+
_api_key: str = None
9+
10+
def update(self):
11+
pass
12+
13+
def getHeaders(self):
14+
return {"x-api-key": self._api_key}

0 commit comments

Comments
 (0)