Added timeout parameter malware bazaar end threatfox (intelowlproject#2691)

* added limit parameter

* fixed reverse migrate

Author: federicofantini

api_app/ingestors_manager/ingestors/malware_bazaar.py

@@ -23,6 +23,8 @@ class MalwareBazaar(AbuseCHMixin, Ingestor):
     hours: int
     # Download samples from chosen signatures (aka malware families)
     signatures: str
+    # Max number of results you want to display
+    limit: int
 
     @classmethod
     def update(cls) -> bool:
@@ -32,7 +34,7 @@ def update(cls) -> bool:
     def get_signature_information(self, signature):
         result = requests.post(
             self.url,
-            data={"query": "get_siginfo", "signature": signature, "limit": 100},
+            data={"query": "get_siginfo", "signature": signature, "limit": self.limit},
             headers=self.authentication_header,
             timeout=30,
         )
@@ -53,20 +55,19 @@ def get_recent_samples(self):
         current_time = timezone.now()
         for signature in self.signatures:
             data = self.get_signature_information(signature)
+            hours_str = "hour" if self.hours == 1 else "hours"
+            if len(data) > self.limit:
+                logger.info(
+                    f"{signature}: in the last {hours_str} there are "
+                    f"more results than the limit {len(data)}/{self.limit}"
+                )
             for elem in data:
                 first_seen = timezone.make_aware(
                     timezone.datetime.strptime(elem["first_seen"], "%Y-%m-%d %H:%M:%S")
                 )
                 diff = int((current_time - first_seen).total_seconds()) // 3600
                 if elem["signature"] == signature and diff <= self.hours:
                     hashes.add(elem["sha256_hash"])
-
-            last_hours_str = (
-                "Last hour" if self.hours == 1 else f"Last {self.hours} hours"
-            )
-            logger.info(
-                f"{last_hours_str} {signature} samples: {len(hashes)}/{len(data)}"
-            )
         return hashes
 
     def download_sample(self, h):

api_app/ingestors_manager/migrations/0027_added_limit_parameter_malware_bazaar_threatfox.py

@@ -0,0 +1,80 @@
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+    IngestorConfig = apps.get_model("ingestors_manager", "IngestorConfig")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+
+    ingestors = [
+        "malware_bazaar.MalwareBazaar",
+        "threatfox.ThreatFox",
+    ]
+    for ingestor in ingestors:
+        module = PythonModule.objects.get(
+            module=ingestor,
+            base_path="api_app.ingestors_manager.ingestors",
+        )
+        p = Parameter.objects.create(
+            name="limit",
+            type="int",
+            description="Max number of results.",
+            is_secret=False,
+            required=True,
+            python_module=module,
+        )
+        p.full_clean()
+        p.save()
+
+        ic = IngestorConfig.objects.get(name=ingestor.split(".")[1])
+        pc = PluginConfig(
+            value=20,
+            ingestor_config=ic,
+            for_organization=False,
+            owner=None,
+            parameter=p,
+        )
+        pc.full_clean()
+        pc.save()
+
+
+def reverse_migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+    IngestorConfig = apps.get_model("ingestors_manager", "IngestorConfig")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+
+    ingestors = [
+        "malware_bazaar.MalwareBazaar",
+        "threatfox.ThreatFox",
+    ]
+    for ingestor in ingestors:
+        module = PythonModule.objects.get(
+            module=ingestor,
+            base_path="api_app.ingestors_manager.ingestors",
+        )
+        ic = IngestorConfig.objects.get(name=ingestor.split(".")[1])
+        p = Parameter.objects.get(
+            name="limit",
+            type="int",
+            description="Max number of results.",
+            is_secret=False,
+            required=True,
+            python_module=module,
+        )
+        PluginConfig.objects.get(
+            parameter=p,
+            ingestor_config=ic,
+        ).delete()
+        p.delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0065_job_mpnodesearch"),
+        ("ingestors_manager", "0026_alter_ingestor_config_malware_bazaar_threatfox"),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]