Major vhost (virtualhost) rewrite

bbot/core/event/base.py

@@ -1604,16 +1604,17 @@ def _pretty_string(self):
         return self.data["technology"]
 
 
-class VHOST(DictHostEvent):
+class VIRTUAL_HOST(DictHostEvent):
     class _data_validator(BaseModel):
         host: str
-        vhost: str
+        virtual_host: str
+        description: str
         url: Optional[str] = None
         _validate_url = field_validator("url")(validators.validate_url)
         _validate_host = field_validator("host")(validators.validate_host)
 
     def _pretty_string(self):
-        return self.data["vhost"]
+        return self.data["virtual_host"]
 
 
 class PROTOCOL(DictHostEvent):

bbot/core/helpers/misc.py

@@ -11,9 +11,11 @@
 import regex as re
 import subprocess as sp
 
+
 from pathlib import Path
 from contextlib import suppress
 from unidecode import unidecode  # noqa F401
+from typing import Iterable, Awaitable, Optional
 from asyncio import create_task, gather, sleep, wait_for  # noqa
 from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin  # noqa F401
 
@@ -2589,30 +2591,55 @@ def parse_port_string(port_string):
     return ports
 
 
-async def as_completed(coros):
+async def as_completed(
+    coroutines: Iterable[Awaitable],
+    max_concurrent: Optional[int] = None,
+):
+    """
+    Yield completed coroutines as they finish with optional concurrency limiting.
+    All coroutines are scheduled as tasks internally for execution.
     """
-    Async generator that yields completed Tasks as they are completed.
+    it = iter(coroutines)
 
-    Args:
-        coros (iterable): An iterable of coroutine objects or asyncio Tasks.
+    # Prime the running set up to the concurrency limit (or all, if unlimited)
+    running = set()
+    limit = max_concurrent or float("inf")
+    try:
+        while len(running) < limit:
+            coro = next(it)
+            running.add(asyncio.create_task(coro))
+    except StopIteration:
+        pass
 
-    Yields:
-        asyncio.Task: A Task object that has completed its execution.
+    # Drain: yield completed tasks, backfill from the iterator as slots free up
+    while running:
+        done, running = await asyncio.wait(running, return_when=asyncio.FIRST_COMPLETED)
+        for task in done:
+            # Immediately backfill one slot per completed task, if more work remains
+            try:
+                coro = next(it)
+                running.add(asyncio.create_task(coro))
+            except StopIteration:
+                pass
+            yield task
 
-    Examples:
-        >>> async def main():
-        ...     async for task in as_completed([coro1(), coro2(), coro3()]):
-        ...         result = task.result()
-        ...         print(f'Task completed with result: {result}')
 
-        >>> asyncio.run(main())
+def get_waf_strings():
     """
-    tasks = {coro if isinstance(coro, asyncio.Task) else asyncio.create_task(coro): coro for coro in coros}
-    while tasks:
-        done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED)
-        for task in done:
-            tasks.pop(task)
-            yield task
+    Returns a list of common WAF (Web Application Firewall) detection strings.
+
+    Returns:
+        list: List of WAF detection strings
+
+    Examples:
+        >>> waf_strings = get_waf_strings()
+        >>> "The requested URL was rejected" in waf_strings
+        True
+    """
+    return [
+        "The requested URL was rejected",
+        "This content has been blocked",
+    ]
 
 
 def clean_dns_record(record):

bbot/core/helpers/web/web.py

@@ -2,6 +2,7 @@
 import warnings
 from pathlib import Path
 from bs4 import BeautifulSoup
+import ipaddress
 
 from bbot.core.engine import EngineClient
 from bbot.core.helpers.misc import truncate_filename
@@ -321,10 +322,11 @@ async def curl(self, *args, **kwargs):
             path_override (str, optional): Overrides the request-target to use in the HTTP request line.
             head_mode (bool, optional): If True, includes '-I' to fetch headers only. Defaults to None.
             raw_body (str, optional): Raw string to be sent in the body of the request.
+            resolve (dict, optional): Host resolution override as dict with 'host', 'port', 'ip' keys for curl --resolve.
             **kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.
 
         Returns:
-            str: The output of the cURL command.
+            dict: JSON object with response data and metadata.
 
         Raises:
             CurlError: If 'url' is not supplied.
@@ -338,7 +340,11 @@ async def curl(self, *args, **kwargs):
         if not url:
             raise CurlError("No URL supplied to CURL helper")
 
-        curl_command = ["curl", url, "-s"]
+        # Use BBOT-specific curl binary
+        bbot_curl = self.parent_helper.tools_dir / "curl"
+        if not bbot_curl.exists():
+            raise CurlError(f"BBOT curl binary not found at {bbot_curl}. Run dependency installation.")
+        curl_command = [str(bbot_curl), url, "-s"]
 
         raw_path = kwargs.get("raw_path", False)
         if raw_path:
@@ -382,6 +388,12 @@ async def curl(self, *args, **kwargs):
         curl_command.append("-m")
         curl_command.append(str(timeout))
 
+        # mirror the web helper behavior
+        retries = self.parent_helper.web_config.get("http_retries", 1)
+        if retries > 0:
+            curl_command.extend(["--retry", str(retries)])
+            curl_command.append("--retry-all-errors")
+
         for k, v in headers.items():
             if isinstance(v, list):
                 for x in v:
@@ -426,9 +438,71 @@ async def curl(self, *args, **kwargs):
         if raw_body:
             curl_command.append("-d")
             curl_command.append(raw_body)
+
+        # --resolve <host>:<port>:<ip>
+        resolve_dict = kwargs.get("resolve", None)
+
+        if resolve_dict is not None:
+            # Validate "resolve" is a dict
+            if not isinstance(resolve_dict, dict):
+                raise CurlError("'resolve' must be a dictionary containing 'host', 'port', and 'ip' keys")
+
+            # Extract and validate IP (required)
+            ip = resolve_dict.get("ip")
+            if not ip:
+                raise CurlError("'resolve' dictionary requires an 'ip' value")
+            try:
+                ipaddress.ip_address(ip)
+            except ValueError:
+                raise CurlError(f"Invalid IP address supplied to 'resolve': {ip}")
+
+            # Host, port, and ip must ALL be supplied explicitly
+            host = resolve_dict.get("host")
+            if not host:
+                raise CurlError("'resolve' dictionary requires a 'host' value")
+
+            if "port" not in resolve_dict:
+                raise CurlError("'resolve' dictionary requires a 'port' value")
+            port = resolve_dict["port"]
+
+            try:
+                port = int(port)
+            except (TypeError, ValueError):
+                raise CurlError("'port' supplied to resolve must be an integer")
+            if port < 1 or port > 65535:
+                raise CurlError("'port' supplied to resolve must be between 1 and 65535")
+
+            # Append the --resolve directive
+            curl_command.append("--resolve")
+            curl_command.append(f"{host}:{port}:{ip}")
+
+        # Always add JSON --write-out format with separator
+        curl_command.extend(["-w", "\\n---CURL_METADATA---\\n%{json}"])
+
         log.verbose(f"Running curl command: {curl_command}")
         output = (await self.parent_helper.run(curl_command)).stdout
-        return output
+
+        # Parse the output to separate content and metadata
+        import json
+
+        parts = output.split("\n---CURL_METADATA---\n")
+
+        # Raise CurlError if separator not found - this indicates a problem with our curl implementation
+        if len(parts) < 2:
+            raise CurlError(f"Curl output missing expected separator. Got: {output[:200]}...")
+
+        response_data = parts[0]
+        # Take the last part as JSON metadata (in case separator appears in content)
+        json_data = parts[-1].strip()
+
+        # Raise CurlError if JSON parsing fails - this indicates a problem with curl's %{json} output
+        try:
+            metadata = json.loads(json_data)
+        except json.JSONDecodeError as e:
+            raise CurlError(f"Failed to parse curl JSON metadata: {e}. JSON data: {json_data[:200]}...")
+
+        # Combine into final JSON structure
+        return {"response_data": response_data, **metadata}
 
     def beautifulsoup(
         self,

bbot/core/shared_deps.py

@@ -173,6 +173,31 @@
     },
 ]
 
+DEP_CURL = [
+    {
+        "name": "Download static curl binary (v8.11.0)",
+        "get_url": {
+            "url": "https://github.com/moparisthebest/static-curl/releases/download/v8.11.0/curl-amd64",
+            "dest": "#{BBOT_TOOLS}/curl",
+            "mode": "0755",
+            "force": True,
+        },
+    },
+    {
+        "name": "Ensure curl binary is executable",
+        "file": {
+            "path": "#{BBOT_TOOLS}/curl",
+            "mode": "0755",
+        },
+    },
+    {
+        "name": "Verify curl binary works",
+        "command": "#{BBOT_TOOLS}/curl --version",
+        "register": "curl_version_output",
+        "changed_when": False,
+    },
+]
+
 DEP_MASSCAN = [
     {
         "name": "install os deps (Debian)",

bbot/modules/baddns.py

@@ -2,7 +2,6 @@
 from baddns.lib.loader import load_signatures
 from .base import BaseModule
 
-import asyncio
 import logging
 
 
@@ -54,8 +53,17 @@ async def setup(self):
         self.debug(f"Enabled BadDNS Submodules: [{','.join(self.enabled_submodules)}]")
         return True
 
+    async def _run_module(self, module_instance):
+        """Wrapper coroutine that runs a module and returns both the module and result"""
+        try:
+            result = await module_instance.dispatch()
+            return module_instance, result
+        except Exception as e:
+            self.warning(f"Task for {module_instance} raised an error: {e}")
+            return module_instance, None
+
     async def handle_event(self, event):
-        tasks = []
+        coroutines = []
         for ModuleClass in self.select_modules():
             kwargs = {
                 "http_client_class": self.scan.helpers.web.AsyncClient,
@@ -70,16 +78,16 @@ async def handle_event(self, event):
                 kwargs["raw_query_retry_wait"] = 0
 
             module_instance = ModuleClass(event.data, **kwargs)
-            task = asyncio.create_task(module_instance.dispatch())
-            tasks.append((module_instance, task))
+            # Create wrapper coroutine that includes the module instance
+            coroutine = self._run_module(module_instance)
+            coroutines.append(coroutine)
 
-        async for completed_task in self.helpers.as_completed([task for _, task in tasks]):
-            module_instance = next((m for m, t in tasks if t == completed_task), None)
+        async for completed_coro in self.helpers.as_completed(coroutines):
             try:
-                task_result = await completed_task
+                module_instance, task_result = await completed_coro
             except Exception as e:
-                self.warning(f"Task for {module_instance} raised an error: {e}")
-                task_result = None
+                self.warning(f"Wrapper coroutine raised an error: {e}")
+                continue
 
             if task_result:
                 results = module_instance.analyze()
@@ -134,4 +142,4 @@ async def handle_event(self, event):
                                     tags=[f"baddns-{module_instance.name.lower()}"],
                                     context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {{event.data}}',
                                 )
-                await module_instance.cleanup()
+            await module_instance.cleanup()

bbot/modules/bypass403.py

@@ -63,8 +63,6 @@
     "X-Host": "127.0.0.1",
 }
 
-# This is planned to be replaced in the future: https://github.com/blacklanternsecurity/bbot/issues/1068
-waf_strings = ["The requested URL was rejected"]
 
 for qp in query_payloads:
     signatures.append(("GET", "{scheme}://{netloc}/{path}%s" % qp, None, True))
@@ -107,8 +105,8 @@ async def do_checks(self, compare_helper, event, collapse_threshold):
 
             # In some cases WAFs will respond with a 200 code which causes a false positive
             if subject_response is not None:
-                for ws in waf_strings:
-                    if ws in subject_response.text:
+                for waf_string in self.helpers.get_waf_strings():
+                    if waf_string in subject_response.text:
                         self.debug("Rejecting result based on presence of WAF string")
                         return
 

bbot/modules/generic_ssrf.py

@@ -39,6 +39,8 @@ class BaseSubmodule:
     severity = "INFO"
     paths = []
 
+    deps_common = ["curl"]
+
     def __init__(self, generic_ssrf):
         self.generic_ssrf = generic_ssrf
         self.test_paths = self.create_paths()
@@ -61,7 +63,7 @@ async def test(self, event):
                 self.generic_ssrf.debug(f"Sending request to URL: {test_url}")
                 r = await self.generic_ssrf.helpers.curl(url=test_url)
                 if r:
-                    self.process(event, r, subdomain_tag)
+                    self.process(event, r["response_data"], subdomain_tag)
 
     def process(self, event, r, subdomain_tag):
         response_token = self.generic_ssrf.interactsh_domain.split(".")[0][::-1]
@@ -123,7 +125,7 @@ async def test(self, event):
 
         for tag, pd in post_data_list:
             r = await self.generic_ssrf.helpers.curl(url=test_url, method="POST", post_data=pd)
-            self.process(event, r, tag)
+            self.process(event, r["response_data"], tag)
 
 
 class Generic_XXE(BaseSubmodule):
@@ -146,7 +148,7 @@ async def test(self, event):
             url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"}
         )
         if r:
-            self.process(event, r, subdomain_tag)
+            self.process(event, r["response_data"], subdomain_tag)
 
 
 class generic_ssrf(BaseModule):