Skip to content

Commit f3ac867

Browse files
pranjalg1331meshery-dev[bot]
pranjalg1331
and
meshery-dev[bot]
authored
Nuclei analyzer (#2697)
* Install Kanvas workflow * initial commit Signed-off-by: pranjalg1331 <[email protected]> * Dockerfile Signed-off-by: pranjalg1331 <[email protected]> * synchronous nuclei api Signed-off-by: pranjalg1331 <[email protected]> * removing unnecessary file Signed-off-by: pranjalg1331 <[email protected]> * dockerfile Signed-off-by: pranjalg1331 <[email protected]> * made api async Signed-off-by: pranjalg1331 <[email protected]> * error resolved Signed-off-by: pranjalg1331 <[email protected]> * async working Signed-off-by: pranjalg1331 <[email protected]> * another Signed-off-by: pranjalg1331 <[email protected]> * deepsource errors Signed-off-by: pranjalg1331 <[email protected]> * gunicorn version fixed Signed-off-by: pranjalg1331 <[email protected]> * shell2http architecture Signed-off-by: pranjalg1331 <[email protected]> * deep sorce errors resolved Signed-off-by: pranjalg1331 <[email protected]> * nuclei file update Signed-off-by: pranjalg1331 <[email protected]> * new version for dockerfile Signed-off-by: pranjalg1331 <[email protected]> * migration file changes Signed-off-by: pranjalg1331 <[email protected]> * error corected Signed-off-by: pranjalg1331 <[email protected]> * error resolved Signed-off-by: pranjalg1331 <[email protected]> * dockerfile cleanup code Signed-off-by: pranjalg1331 <[email protected]> * dependabot added + dockerfile test version Signed-off-by: pranjalg1331 <[email protected]> * dependabot for python dependencies Signed-off-by: pranjalg1331 <[email protected]> * healthcheck for api Signed-off-by: pranjalg1331 <[email protected]> --------- Signed-off-by: pranjalg1331 <[email protected]> Co-authored-by: meshery-dev[bot] <132387951+meshery-dev[bot]@users.noreply.github.com>
1 parent 955e3e4 commit f3ac867

File tree

10 files changed

+422
-1
lines changed

10 files changed

+422
-1
lines changed

‎.github/dependabot.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,18 @@ updates:
5252
- dependency-name: "*"
5353
update-types: [ "version-update:semver-patch" ]
5454

55+
- package-ecosystem: "pip"
56+
directory: "/integrations/nuclei_analyzer"
57+
schedule:
58+
interval: "weekly"
59+
day: "tuesday"
60+
target-branch: "develop"
61+
ignore:
62+
# ignore all patch updates since we are using ~=
63+
# this does not work for security updates
64+
- dependency-name: "*"
65+
update-types: [ "version-update:semver-patch" ]
66+
5567
- package-ecosystem: "pip"
5668
directory: "/integrations/phishing_analyzers"
5769
schedule:
@@ -121,6 +133,18 @@ updates:
121133
- dependency-name: "*"
122134
update-types: ["version-update:semver-patch"]
123135

136+
- package-ecosystem: "docker"
137+
directory: "/integrations/nuclei_analyzer"
138+
schedule:
139+
interval: "weekly"
140+
day: "tuesday"
141+
target-branch: "develop"
142+
ignore:
143+
# ignore all patch updates since we are using ~=
144+
# this does not work for security updates
145+
- dependency-name: "*"
146+
update-types: ["version-update:semver-patch"]
147+
124148
- package-ecosystem: "docker"
125149
directory: "/integrations/malware_tools_analyzers"
126150
schedule:

‎api_app/analyzers_manager/migrations/0148_analyzer_config_nuclei.py‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
from django.db import migrations
2+
from django.db.models.fields.related_descriptors import (
3+
ForwardManyToOneDescriptor,
4+
ForwardOneToOneDescriptor,
5+
ManyToManyDescriptor,
6+
ReverseManyToOneDescriptor,
7+
ReverseOneToOneDescriptor,
8+
)
9+
10+
plugin = {
11+
"python_module": {
12+
"health_check_schedule": None,
13+
"update_schedule": None,
14+
"module": "nuclei.NucleiAnalyzer",
15+
"base_path": "api_app.analyzers_manager.observable_analyzers",
16+
},
17+
"name": "Nuclei",
18+
"description": "[Nuclei](https://github.com/projectdiscovery/nuclei) is a fast, customizable vulnerability scanner that leverages YAML-based templates to detect, rank, and address security flaws. It operates using structured templates that define specific security checks.",
19+
"disabled": False,
20+
"soft_time_limit": 1200,
21+
"routing_key": "default",
22+
"health_check_status": True,
23+
"type": "observable",
24+
"docker_based": True,
25+
"maximum_tlp": "RED",
26+
"observable_supported": ["ip", "url"],
27+
"supported_filetypes": [],
28+
"run_hash": False,
29+
"run_hash_type": "",
30+
"not_supported_filetypes": [],
31+
"mapping_data_model": {},
32+
"model": "analyzers_manager.AnalyzerConfig",
33+
}
34+
35+
params = [
36+
{
37+
"python_module": {
38+
"module": "nuclei.NucleiAnalyzer",
39+
"base_path": "api_app.analyzers_manager.observable_analyzers",
40+
},
41+
"name": "template_dirs",
42+
"type": "list",
43+
"description": "The template_dirs parameter allows you to specify a list of directories containing templates, each focusing on a particular category of vulnerabilities, exposures, or security assessments.\r\nAvailable Template Categories:\r\ncloud\r\ncode\r\ncves\r\nvulnerabilities\r\ndns\r\nfile\r\nheadless\r\nhelpers\r\nhttp\r\njavascript\r\nnetwork\r\npassive\r\nprofiles\r\nssl\r\nworkflows\r\nexposures",
44+
"is_secret": False,
45+
"required": False,
46+
}
47+
]
48+
49+
values = [
50+
{
51+
"parameter": {
52+
"python_module": {
53+
"module": "nuclei.NucleiAnalyzer",
54+
"base_path": "api_app.analyzers_manager.observable_analyzers",
55+
},
56+
"name": "template_dirs",
57+
"type": "list",
58+
"description": "The template_dirs parameter allows you to specify a list of directories containing templates, each focusing on a particular category of vulnerabilities, exposures, or security assessments.\r\nAvailable Template Categories:\r\ncloud\r\ncode\r\ncves\r\nvulnerabilities\r\ndns\r\nfile\r\nheadless\r\nhelpers\r\nhttp\r\njavascript\r\nnetwork\r\npassive\r\nprofiles\r\nssl\r\nworkflows\r\nexposures",
59+
"is_secret": False,
60+
"required": False,
61+
},
62+
"analyzer_config": "Nuclei",
63+
"connector_config": None,
64+
"visualizer_config": None,
65+
"ingestor_config": None,
66+
"pivot_config": None,
67+
"for_organization": False,
68+
"value": [],
69+
"updated_at": "2025-01-08T08:33:45.653741Z",
70+
"owner": None,
71+
}
72+
]
73+
74+
75+
def _get_real_obj(Model, field, value):
76+
def _get_obj(Model, other_model, value):
77+
if isinstance(value, dict):
78+
real_vals = {}
79+
for key, real_val in value.items():
80+
real_vals[key] = _get_real_obj(other_model, key, real_val)
81+
value = other_model.objects.get_or_create(**real_vals)[0]
82+
# it is just the primary key serialized
83+
else:
84+
if isinstance(value, int):
85+
if Model.__name__ == "PluginConfig":
86+
value = other_model.objects.get(name=plugin["name"])
87+
else:
88+
value = other_model.objects.get(pk=value)
89+
else:
90+
value = other_model.objects.get(name=value)
91+
return value
92+
93+
if (
94+
type(getattr(Model, field))
95+
in [
96+
ForwardManyToOneDescriptor,
97+
ReverseManyToOneDescriptor,
98+
ReverseOneToOneDescriptor,
99+
ForwardOneToOneDescriptor,
100+
]
101+
and value
102+
):
103+
other_model = getattr(Model, field).get_queryset().model
104+
value = _get_obj(Model, other_model, value)
105+
elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
106+
other_model = getattr(Model, field).rel.model
107+
value = [_get_obj(Model, other_model, val) for val in value]
108+
return value
109+
110+
111+
def _create_object(Model, data):
112+
mtm, no_mtm = {}, {}
113+
for field, value in data.items():
114+
value = _get_real_obj(Model, field, value)
115+
if type(getattr(Model, field)) is ManyToManyDescriptor:
116+
mtm[field] = value
117+
else:
118+
no_mtm[field] = value
119+
try:
120+
o = Model.objects.get(**no_mtm)
121+
except Model.DoesNotExist:
122+
o = Model(**no_mtm)
123+
o.full_clean()
124+
o.save()
125+
for field, value in mtm.items():
126+
attribute = getattr(o, field)
127+
if value is not None:
128+
attribute.set(value)
129+
return False
130+
return True
131+
132+
133+
def migrate(apps, schema_editor):
134+
Parameter = apps.get_model("api_app", "Parameter")
135+
PluginConfig = apps.get_model("api_app", "PluginConfig")
136+
python_path = plugin.pop("model")
137+
Model = apps.get_model(*python_path.split("."))
138+
if not Model.objects.filter(name=plugin["name"]).exists():
139+
exists = _create_object(Model, plugin)
140+
if not exists:
141+
for param in params:
142+
_create_object(Parameter, param)
143+
for value in values:
144+
_create_object(PluginConfig, value)
145+
146+
147+
def reverse_migrate(apps, schema_editor):
148+
python_path = plugin.pop("model")
149+
Model = apps.get_model(*python_path.split("."))
150+
Model.objects.get(name=plugin["name"]).delete()
151+
152+
153+
class Migration(migrations.Migration):
154+
atomic = False
155+
dependencies = [
156+
("api_app", "0065_job_mpnodesearch"),
157+
(
158+
"analyzers_manager",
159+
"0147_alter_analyzer_config_feodo_yaraify_urlhaus_yaraify_scan",
160+
),
161+
]
162+
163+
operations = [migrations.RunPython(migrate, reverse_migrate)]

‎api_app/analyzers_manager/observable_analyzers/nuclei.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
2+
# See the file 'LICENSE' for copying permission.
3+
import logging
4+
5+
from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer
6+
7+
logger = logging.getLogger(__name__)
8+
9+
10+
class NucleiAnalyzer(ObservableAnalyzer, DockerBasedAnalyzer):
11+
url: str = "http://nuclei_analyzer:4008/run-nuclei"
12+
template_dirs: list
13+
max_tries: int = 40
14+
poll_distance: int = 30
15+
16+
@classmethod
17+
def update(cls) -> bool:
18+
pass
19+
20+
def run(self):
21+
"""
22+
Prepares and executes a Nuclei scan through the Docker-based API.
23+
"""
24+
VALID_TEMPLATE_CATEGORIES = {
25+
"cloud",
26+
"code",
27+
"cves",
28+
"vulnerabilities",
29+
"dns",
30+
"file",
31+
"headless",
32+
"helpers",
33+
"http",
34+
"javascript",
35+
"network",
36+
"passive",
37+
"profiles",
38+
"ssl",
39+
"workflows",
40+
"exposures",
41+
}
42+
43+
args = [self.observable_name]
44+
45+
# Append valid template directories with the "-t" flag
46+
for template_dir in self.template_dirs:
47+
if template_dir in VALID_TEMPLATE_CATEGORIES:
48+
args.extend(["-t", template_dir])
49+
else:
50+
warning = f"Skipping invalid template directory: {template_dir} for observable {self.observable_name}"
51+
logger.warning(warning)
52+
self.report.errors.append(warning)
53+
req_data = {"args": args}
54+
55+
# Execute the request
56+
response = self._docker_run(req_data=req_data, req_files=None)
57+
58+
analysis = response.get("data", [])
59+
60+
return analysis

‎integrations/nuclei_analyzer/Dockerfile‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
FROM projectdiscovery/nuclei:v3.3.8
2+
3+
ENV LOG_PATH=/var/log/intel_owl/nuclei_analyzer
4+
ENV USER=nuclei-user
5+
ENV PROJECT_PATH=/app
6+
7+
# Create non-root user
8+
RUN adduser -D -h /home/${USER} ${USER}
9+
10+
# Install required packages using apk and clean cache in the same layer
11+
RUN apk add --no-cache python3=3.11.11-r0 py3-pip \
12+
&& rm -rf /var/cache/apk/* \
13+
&& pip3 install --no-cache-dir --upgrade pip
14+
15+
# Create working directory and set ownership
16+
WORKDIR /app
17+
18+
# Copy and install requirements first (better layer caching)
19+
COPY requirements.txt .
20+
RUN pip3 install --no-cache-dir -r requirements.txt \
21+
&& rm -rf ~/.cache/pip/*
22+
23+
# Create log directory with proper permissions
24+
RUN mkdir -p ${LOG_PATH} \
25+
&& touch ${LOG_PATH}/gunicorn_access.log ${LOG_PATH}/gunicorn_errors.log \
26+
&& chown -R ${USER}:${USER} ${LOG_PATH} \
27+
&& chmod 755 ${LOG_PATH} \
28+
&& chmod 666 ${LOG_PATH}/gunicorn_access.log \
29+
&& chmod 666 ${LOG_PATH}/gunicorn_errors.log
30+
# Copy application files
31+
COPY app.py .
32+
COPY entrypoint.sh /entrypoint.sh
33+
34+
# Set proper permissions
35+
RUN chown -R ${USER}:${USER} /app \
36+
&& chmod +x /entrypoint.sh
37+
38+
# Expose the API port
39+
EXPOSE 4008
40+
41+
HEALTHCHECK --interval=45s --timeout=10s --retries=3 \
42+
CMD curl -f http://localhost:4008/health || exit 1
43+
44+
ENTRYPOINT ["/entrypoint.sh"]

‎integrations/nuclei_analyzer/app.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import json
2+
import logging
3+
import os
4+
5+
from flask import Flask
6+
from flask_executor import Executor
7+
from flask_shell2http import Shell2HTTP
8+
9+
# Logger configuration
10+
LOG_NAME = "nuclei_scanner"
11+
logger = logging.getLogger("flask_shell2http")
12+
13+
# Create formatter
14+
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
15+
16+
# Set log level from environment variable or default to INFO
17+
log_level = os.getenv("LOG_LEVEL", logging.INFO)
18+
log_path = os.getenv("LOG_PATH", f"/var/log/intel_owl/{LOG_NAME}")
19+
20+
# Create file handlers for both general logs and errors
21+
fh = logging.FileHandler(f"{log_path}/{LOG_NAME}.log")
22+
fh.setFormatter(formatter)
23+
fh.setLevel(log_level)
24+
25+
fh_err = logging.FileHandler(f"{log_path}/{LOG_NAME}_errors.log")
26+
fh_err.setFormatter(formatter)
27+
fh_err.setLevel(logging.ERROR)
28+
29+
# Add handlers to logger
30+
logger.addHandler(fh)
31+
logger.addHandler(fh_err)
32+
logger.setLevel(log_level)
33+
34+
# Flask application instance with secret key
35+
app = Flask(__name__)
36+
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY", os.urandom(24).hex())
37+
38+
# Initialize the Executor for background task processing
39+
executor = Executor(app)
40+
41+
# Initialize the Shell2HTTP for exposing shell commands as HTTP endpoints
42+
shell2http = Shell2HTTP(app=app, executor=executor)
43+
44+
45+
@app.route("/health", methods=["GET"])
46+
def health_check():
47+
return {"status": "healthy"}, 200
48+
49+
50+
def my_callback_fn(context, future):
51+
"""
52+
Callback function to handle Nuclei scan results
53+
"""
54+
try:
55+
result = future.result()
56+
report = result["report"]
57+
# The report is a string with multiple JSON objects separated by newlines
58+
json_objects = []
59+
for line in report.strip().split("\n"):
60+
try:
61+
json_objects.append(json.loads(line))
62+
except json.JSONDecodeError:
63+
logger.warning(f"Skipping non-JSON line: {line}")
64+
result["report"] = {"data": json_objects}
65+
logger.info(f"Nuclei scan completed for context: {context}")
66+
logger.debug(f"Scan result: {result}")
67+
except Exception as e:
68+
logger.error(f"Error in callback function: {str(e)}", exc_info=True)
69+
raise
70+
71+
72+
# Register the 'nuclei' command
73+
shell2http.register_command(
74+
endpoint="run-nuclei",
75+
command_name="nuclei -j -ud /opt/nuclei-api/nuclei-templates -u",
76+
callback_fn=my_callback_fn,
77+
)
78+
79+
80+
if __name__ == "__main__":
81+
logger.info("Starting Nuclei scanner API server")
82+
app.run(host="0.0.0.0", port=4008)

‎integrations/nuclei_analyzer/compose-tests.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
services:
2+
nuclei_analyzer:
3+
build:
4+
context: ../integrations/nuclei_analyzer
5+
dockerfile: Dockerfile
6+
image: intelowlproject/nuclei_analyzer:test

0 commit comments

Comments
 (0)