feat: remove duplicated alerts

Sometimes the different client tools generate multiple requests when
the user requests a task. This generates what it looks as a duplicate
alert, but is not really a total duplicate as it belongs to different
request.
But for the user does not provide so much value to have it, so proceed
with deduplicating those alerts, based on the code snippet and details
of the alert

Also remove dogecoin regex as it's giving false positives

Closes: #875

Author: yrobla

signatures.yaml

@@ -281,7 +281,6 @@
     - Bitcoin SegWit: \b(bc1)[a-zA-HJ-NP-Z0-9]{39,59}\b
     - Ethereum: \b0x[a-fA-F0-9]{40}\b
     - Litecoin: \b(L|M)[a-km-zA-HJ-NP-Z1-9]{26,33}\b
-    - Dogecoin: \b(D|A)[a-km-zA-HJ-NP-Z1-9]{25,34}\b
     - Ripple: \br[rK][a-zA-Z0-9]{25,35}\b
     - Monero: \b4[0-9AB][1-9A-HJ-NP-Za-km-z]{93}\b
     - Tron: \bT[a-zA-HJ-NP-Z0-9]{33}\b

src/codegate/api/v1.py

@@ -390,6 +390,7 @@ async def get_workspace_alerts(workspace_name: str) -> List[Optional[v1_models.A
 
     try:
         alerts = await dbreader.get_alerts_by_workspace(ws.id, AlertSeverity.CRITICAL.value)
+        alerts = v1_processing.remove_duplicate_alerts(alerts)
         prompts_outputs = await dbreader.get_prompts_with_output(ws.id)
         return await v1_processing.parse_get_alert_conversation(alerts, prompts_outputs)
     except Exception:

src/codegate/api/v1_processing.py

@@ -62,7 +62,7 @@ async def _is_system_prompt(message: str) -> bool:
     return False
 
 
-async def parse_request(request_str: str) -> Tuple[Optional[List[str]], str]:
+async def parse_request(request_str: str) -> Tuple[Optional[List[str]], str]:  # noqa: C901
     """
     Parse the request string from the pipeline and return the message and the model.
     """
@@ -105,7 +105,7 @@ async def parse_request(request_str: str) -> Tuple[Optional[List[str]], str]:
     return messages, model
 
 
-async def parse_output(output_str: str) -> Optional[str]:
+async def parse_output(output_str: str) -> Optional[str]:  # noqa: C901
     """
     Parse the output string from the pipeline and return the message.
     """
@@ -499,3 +499,37 @@ async def parse_workspace_token_usage(
     for p_qa in partial_question_answers:
         token_usage_agg.add_model_token_usage(p_qa.model_token_usage)
     return token_usage_agg
+
+
+def remove_duplicate_alerts(alerts):
+    unique_alerts = []
+    seen = defaultdict(list)
+
+    for alert in sorted(
+        alerts, key=lambda x: x.timestamp, reverse=True
+    ):  # Sort alerts by timestamp descending
+        if alert.trigger_type != "codegate-secrets":
+            unique_alerts.append(alert)
+            continue
+
+        # Extract trigger string content until "Context"
+        trigger_string_content = alert.trigger_string.split("Context")[0]
+
+        key = (
+            alert.code_snippet,
+            alert.trigger_type,
+            alert.trigger_category,
+            trigger_string_content,
+        )
+
+        # If key exists and new alert is more recent, replace it
+        if key in seen:
+            existing_alert = seen[key]
+            if abs((alert.timestamp - existing_alert.timestamp).total_seconds()) < 5:
+                seen[key] = alert  # Replace with newer alert
+                continue
+
+        seen[key] = alert
+        unique_alerts.append(alert)
+
+    return list(seen.values())