Skip to content

Commit 3698116

Browse files
AnshSinghalfgibertoni
AnshSinghal
and
fgibertoni
authored
[Analyzer] Bbot (#2773)
* initial code and setiings for BBOT * testing different methods for BBOT * tried changing permissions * tried changing permissions * tried changing permissions * made a docker based analyzer BBOT * switched from Flask to quart * switched from quart to fastapi * Completed BBOT Analyzer * Removed Entrypoint.sh * fixed monkeypatch method * fixed the error * fixed the tests error * updated classes.py * updated classes.py * removed monkeypatch * added monkeypatch and fixed docker compose files * updated the start file for bbot analyzer * changed the port * tried changing the workflow for bbot (will revert) * updated the conf file * tried changing the workflow for bbot (will revert) * tried adding logs for bbot * tried adding logs for bbot * tried adding logs for bbot * after logs for bbot * after logs for bbot for debugging * changed the monkeypatch method * reverted workflow file * reverted workflow file * added comment, changed function name, updated requirements * updated requirements * improved error handling and fixed some issues * fixed some minor issues * fixed the requested changes * Update integrations/bbot/app.py Co-authored-by: Federico Gibertoni <[email protected]> --------- Co-authored-by: Federico Gibertoni <[email protected]>
1 parent 71f0bb7 commit 3698116

File tree

10 files changed

+474
-10
lines changed

10 files changed

+474
-10
lines changed

‎.github/workflows/pull_request_automation.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,4 +155,4 @@ jobs:
155155
- name: Test with Jest
156156
run: |
157157
npm run test -- --silent --coverage
158-
working-directory: ./frontend
158+
working-directory: ./frontend

‎api_app/analyzers_manager/classes.py‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -433,14 +433,19 @@ def _docker_run(
433433
self._raise_container_not_running()
434434

435435
# step #2: raise AnalyzerRunException in case of error
436-
if not self.__raise_in_case_bad_request(self.name, resp1):
437-
raise AssertionError
438-
439-
# step #3: if no error, continue and try to fetch result
440-
key = resp1.json().get("key")
441-
final_resp = self.__poll_for_result(key)
442-
err = final_resp.get("error", None)
443-
report = final_resp.get("report", None)
436+
# Modified to support synchronous analyzer BBOT that return results directly in the initial response, avoiding unnecessary polling.
437+
if analyzer_name == "BBOT_Analyzer":
438+
report = resp1.json().get("report", None)
439+
err = resp1.json().get("error", None)
440+
else:
441+
if not self.__raise_in_case_bad_request(self.name, resp1):
442+
raise AssertionError
443+
444+
# step #3: if no error, continue and try to fetch result
445+
key = resp1.json().get("key")
446+
final_resp = self.__poll_for_result(key)
447+
err = final_resp.get("error", None)
448+
report = final_resp.get("report", None)
444449

445450
# APKiD provides empty result in case it does not support the binary type
446451
if not report and (analyzer_name != "APKiD"):

‎api_app/analyzers_manager/migrations/0154_analyzer_config_bbot.py‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
from django.db import migrations
2+
from django.db.models.fields.related_descriptors import (
3+
ForwardManyToOneDescriptor,
4+
ForwardOneToOneDescriptor,
5+
ManyToManyDescriptor,
6+
ReverseManyToOneDescriptor,
7+
ReverseOneToOneDescriptor,
8+
)
9+
10+
plugin = {
11+
"python_module": {
12+
"health_check_schedule": None,
13+
"update_schedule": None,
14+
"module": "bbot.BBOT",
15+
"base_path": "api_app.analyzers_manager.observable_analyzers",
16+
},
17+
"name": "BBOT",
18+
"description": "[BBOT](https://github.com/blacklanternsecurity/bbot) (Bighuge BLS Open Threat) domain/URL scanner.\r\nLeverages BBOT's Python library to perform scans with configurable modules and presets.",
19+
"disabled": False,
20+
"soft_time_limit": 600,
21+
"routing_key": "default",
22+
"health_check_status": True,
23+
"type": "observable",
24+
"docker_based": True,
25+
"maximum_tlp": "CLEAR",
26+
"observable_supported": ["ip", "url", "domain"],
27+
"supported_filetypes": [],
28+
"run_hash": False,
29+
"run_hash_type": "",
30+
"not_supported_filetypes": [],
31+
"mapping_data_model": {},
32+
"model": "analyzers_manager.AnalyzerConfig",
33+
}
34+
35+
params = [
36+
{
37+
"python_module": {
38+
"module": "bbot.BBOT",
39+
"base_path": "api_app.analyzers_manager.observable_analyzers",
40+
},
41+
"name": "modules",
42+
"type": "list",
43+
"description": "",
44+
"is_secret": False,
45+
"required": False,
46+
},
47+
{
48+
"python_module": {
49+
"module": "bbot.BBOT",
50+
"base_path": "api_app.analyzers_manager.observable_analyzers",
51+
},
52+
"name": "presets",
53+
"type": "list",
54+
"description": "",
55+
"is_secret": False,
56+
"required": False,
57+
},
58+
]
59+
60+
values = [
61+
{
62+
"parameter": {
63+
"python_module": {
64+
"module": "bbot.BBOT",
65+
"base_path": "api_app.analyzers_manager.observable_analyzers",
66+
},
67+
"name": "modules",
68+
"type": "list",
69+
"description": "",
70+
"is_secret": False,
71+
"required": False,
72+
},
73+
"analyzer_config": "BBOT",
74+
"connector_config": None,
75+
"visualizer_config": None,
76+
"ingestor_config": None,
77+
"pivot_config": None,
78+
"for_organization": False,
79+
"value": [],
80+
"updated_at": "2025-03-08T13:26:48.196551Z",
81+
"owner": None,
82+
},
83+
{
84+
"parameter": {
85+
"python_module": {
86+
"module": "bbot.BBOT",
87+
"base_path": "api_app.analyzers_manager.observable_analyzers",
88+
},
89+
"name": "presets",
90+
"type": "list",
91+
"description": "",
92+
"is_secret": False,
93+
"required": False,
94+
},
95+
"analyzer_config": "BBOT",
96+
"connector_config": None,
97+
"visualizer_config": None,
98+
"ingestor_config": None,
99+
"pivot_config": None,
100+
"for_organization": False,
101+
"value": ["web-basic"],
102+
"updated_at": "2025-03-08T13:26:48.116399Z",
103+
"owner": None,
104+
},
105+
]
106+
107+
108+
def _get_real_obj(Model, field, value):
109+
def _get_obj(Model, other_model, value):
110+
if isinstance(value, dict):
111+
real_vals = {}
112+
for key, real_val in value.items():
113+
real_vals[key] = _get_real_obj(other_model, key, real_val)
114+
value = other_model.objects.get_or_create(**real_vals)[0]
115+
# it is just the primary key serialized
116+
else:
117+
if isinstance(value, int):
118+
if Model.__name__ == "PluginConfig":
119+
value = other_model.objects.get(name=plugin["name"])
120+
else:
121+
value = other_model.objects.get(pk=value)
122+
else:
123+
value = other_model.objects.get(name=value)
124+
return value
125+
126+
if (
127+
type(getattr(Model, field))
128+
in [
129+
ForwardManyToOneDescriptor,
130+
ReverseManyToOneDescriptor,
131+
ReverseOneToOneDescriptor,
132+
ForwardOneToOneDescriptor,
133+
]
134+
and value
135+
):
136+
other_model = getattr(Model, field).get_queryset().model
137+
value = _get_obj(Model, other_model, value)
138+
elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
139+
other_model = getattr(Model, field).rel.model
140+
value = [_get_obj(Model, other_model, val) for val in value]
141+
return value
142+
143+
144+
def _create_object(Model, data):
145+
mtm, no_mtm = {}, {}
146+
for field, value in data.items():
147+
value = _get_real_obj(Model, field, value)
148+
if type(getattr(Model, field)) is ManyToManyDescriptor:
149+
mtm[field] = value
150+
else:
151+
no_mtm[field] = value
152+
try:
153+
o = Model.objects.get(**no_mtm)
154+
except Model.DoesNotExist:
155+
o = Model(**no_mtm)
156+
o.full_clean()
157+
o.save()
158+
for field, value in mtm.items():
159+
attribute = getattr(o, field)
160+
if value is not None:
161+
attribute.set(value)
162+
return False
163+
return True
164+
165+
166+
def migrate(apps, schema_editor):
167+
Parameter = apps.get_model("api_app", "Parameter")
168+
PluginConfig = apps.get_model("api_app", "PluginConfig")
169+
python_path = plugin.pop("model")
170+
Model = apps.get_model(*python_path.split("."))
171+
if not Model.objects.filter(name=plugin["name"]).exists():
172+
exists = _create_object(Model, plugin)
173+
if not exists:
174+
for param in params:
175+
_create_object(Parameter, param)
176+
for value in values:
177+
_create_object(PluginConfig, value)
178+
179+
180+
def reverse_migrate(apps, schema_editor):
181+
python_path = plugin.pop("model")
182+
Model = apps.get_model(*python_path.split("."))
183+
Model.objects.get(name=plugin["name"]).delete()
184+
185+
186+
class Migration(migrations.Migration):
187+
atomic = False
188+
dependencies = [
189+
("api_app", "0071_delete_last_elastic_report"),
190+
("analyzers_manager", "0153_alter_spamhaus_drop_supported_observable"),
191+
]
192+
193+
operations = [migrations.RunPython(migrate, reverse_migrate)]

‎api_app/analyzers_manager/observable_analyzers/bbot.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
2+
# See the file 'LICENSE' for copying permission.
3+
import logging
4+
from urllib.parse import urlparse
5+
6+
import requests
7+
8+
from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer
9+
from api_app.analyzers_manager.exceptions import AnalyzerRunException
10+
from api_app.choices import Classification
11+
from api_app.models import PythonConfig
12+
from tests.mock_utils import MockUpResponse
13+
14+
logging.basicConfig(level=logging.DEBUG)
15+
logger = logging.getLogger(__name__)
16+
17+
18+
class BBOT(ObservableAnalyzer, DockerBasedAnalyzer):
19+
"""
20+
BBOT Docker-based analyzer for IntelOwl.
21+
"""
22+
23+
name: str = "BBOT_Analyzer"
24+
url: str = "http://bbot_analyzer:5001/run"
25+
max_tries: int = 25
26+
poll_distance: int = 5
27+
28+
def __init__(self, config: PythonConfig, **kwargs):
29+
super().__init__(config, **kwargs)
30+
self.args: list[str] = []
31+
32+
def config(self, runtime_configuration: dict):
33+
super().config(runtime_configuration)
34+
target = self.observable_name
35+
36+
if self.observable_classification == Classification.URL:
37+
logger.debug(f"Extracting hostname from URL: {target}")
38+
target = urlparse(target).hostname
39+
40+
self.args.append(f"-t {target}")
41+
self.args.extend([f"-p {preset}" for preset in self.presets])
42+
self.args.extend([f"-m {module}" for module in self.modules])
43+
44+
def run(self):
45+
"""
46+
Executes BBOT inside the Docker container via HTTP API.
47+
"""
48+
req_data = {
49+
"target": self.observable_name,
50+
"presets": self.presets,
51+
"modules": self.modules,
52+
}
53+
54+
logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}")
55+
56+
try:
57+
report = self._docker_run(req_data, analyzer_name=self.name)
58+
logger.info(f"BBOT scan completed successfully with report: {report}")
59+
return report
60+
except requests.RequestException as e:
61+
logger.error(f"BBOT HTTP request failed: {e}")
62+
raise AnalyzerRunException(f"Network error contacting BBOT container: {e}")
63+
64+
@classmethod
65+
def update(cls):
66+
pass
67+
68+
@staticmethod
69+
def mocked_docker_analyzer_post(*args, **kwargs):
70+
mock_response = {
71+
"success": True,
72+
"report": {
73+
"events": [
74+
{
75+
"id": "SCAN:7804fe5d0d26eec716926da9a4002d4ceb171300",
76+
"name": "melodramatic_todd",
77+
"preset": {
78+
"flags": ["iis-shortnames", "web-basic"],
79+
"config": {
80+
"modules": {"iis_shortnames": {"detect_only": False}}
81+
},
82+
"description": "melodramatic_todd",
83+
"output_modules": ["json"],
84+
},
85+
"status": "FINISHED",
86+
"target": {
87+
"hash": "a2d3b5795582da7a4edc56ef63ae6d6866a70d9c",
88+
"seeds": ["test.com"],
89+
"blacklist": [],
90+
"seed_hash": "1f26e4e291bfa260f77d2411c88906aee99786c5",
91+
"whitelist": ["test.com"],
92+
"scope_hash": "86df039469ae73720ac0d8cdd7cf92c3953659b4",
93+
"strict_scope": False,
94+
"blacklist_hash": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
95+
"whitelist_hash": "1f26e4e291bfa260f77d2411c88906aee99786c5",
96+
},
97+
"duration": "52 seconds",
98+
"started_at": "2025-03-18T14:30:59.131139",
99+
"finished_at": "2025-03-18T14:31:51.854936",
100+
"duration_seconds": 52.723797,
101+
}
102+
],
103+
"json_output": [],
104+
},
105+
}
106+
return MockUpResponse(mock_response, 200)

‎integrations/bbot/Dockerfile‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
FROM python:3.12-slim
2+
3+
# Environment variables
4+
ENV PROJECT_PATH=/opt/deploy/bbot
5+
ENV HOME=${PROJECT_PATH}
6+
ENV BBOT_HOME=${PROJECT_PATH}
7+
8+
# Install system dependencies
9+
RUN apt-get update && apt-get install -y --no-install-recommends \
10+
build-essential libssl-dev libffi-dev && \
11+
apt-get clean && apt-get autoremove -y && \
12+
rm -rf /var/lib/apt/lists/* /tmp/* /usr/share/doc/* /usr/share/man/*
13+
14+
# Set up project directory
15+
WORKDIR ${PROJECT_PATH}
16+
17+
# Copy application files and requirements
18+
COPY requirements.txt app.py ./
19+
20+
# Upgrade pip and install Python packages
21+
RUN pip install --no-cache-dir --upgrade pip && \
22+
pip install --no-cache-dir -r requirements.txt
23+
24+
# Pre-install BBOT dependencies & Make script executable
25+
RUN bbot --install-all-deps -y --force && \
26+
chmod u+x app.py
27+
28+
# Expose port
29+
EXPOSE 5001
30+
31+
# Entrypoint
32+
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5001", "--log-level", "debug"]

0 commit comments

Comments
 (0)