v6.2.1

.github/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 [**Upgrade Guide**](https://intelowlproject.github.io/docs/IntelOwl/installation/#update-to-the-most-recent-version)
 
+## [v6.2.1](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.2.1)
+Minor fixes and dependencies upgrades
+
 ## [v6.2.0](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.2.0)
 
 Happy new year! :sparkler: And Happy Birthday IntelOwl! :tada: 

.github/workflows/codeql-analysis.yml

@@ -37,14 +37,14 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4.1.0
+      uses: actions/checkout@v4.2.2
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
         fetch-depth: 2
 
     - name: Set up Python
-      uses: actions/setup-python@v5.1.0
+      uses: actions/setup-python@v5.3.0
       with:
         python-version: '3.11'
 
@@ -65,7 +65,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v3.28.0
       with:
         languages: python
         # Override the default behavior so that the action doesn't attempt
@@ -93,4 +93,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v3.28.0

.github/workflows/dependency_review.yml

@@ -11,6 +11,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.2.2
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@v4

.github/workflows/pull_request_automation.yml

@@ -16,10 +16,10 @@ jobs:
     outputs:
       frontend: ${{steps.diff_check.outputs.frontend}}
     steps:
-      - uses: actions/checkout@v4.1.0
+      - uses: actions/checkout@v4.2.2
         with:
           ref: ${{ github.base_ref }}
-      - uses: actions/checkout@v4.1.0
+      - uses: actions/checkout@v4.2.2
         with:
           clean: false
       - name: Generate diffs
@@ -34,10 +34,10 @@ jobs:
 
     steps:
       - name: Checkout IntelOwl
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.2.2
 
       - name: Set up Python
-        uses: actions/setup-python@v5.1.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: 3.11
 
@@ -69,7 +69,7 @@ jobs:
           access_token: ${{ github.token }}
 
       - name: Checkout IntelOwl
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.2.2
 
       - name: Prepare Launch
         run: |
@@ -119,7 +119,7 @@ jobs:
     if: ${{ needs.detect-changes.outputs.frontend > 0 }}
 
     steps:
-      - uses: actions/checkout@v4.1.0
+      - uses: actions/checkout@v4.2.2
       - name: Set up NodeJS
         uses: actions/setup-node@v4
         with:

.github/workflows/scorecard.yml

@@ -32,12 +32,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@v4.1.0 # v3.1.0
+        uses: actions/checkout@v4.2.2 # v3.1.0
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0
+        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
         with:
           results_file: results.sarif
           results_format: sarif
@@ -59,7 +59,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: SARIF file
           path: results.sarif
@@ -68,6 +68,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@03e7845b7bfcd5e7fb63d1ae8c61b0e791134fab # v2.22.11
+        uses: github/codeql-action/upload-sarif@5b6e617dc0241b2d60c2bccea90c56b67eceb797 # v2.22.11
         with:
           sarif_file: results.sarif

api_app/investigations_manager/models.py

@@ -1,8 +1,10 @@
+import logging
 from datetime import datetime
 from typing import List
 
 from django.conf import settings
 from django.db import models
+from django.db.models import QuerySet
 
 from api_app.choices import TLP
 from api_app.interfaces import OwnershipAbstractModel
@@ -11,8 +13,11 @@
 from api_app.models import ListCachable
 from certego_saas.apps.user.models import User
 
+logger = logging.getLogger(__name__)
+
 
 class Investigation(OwnershipAbstractModel, ListCachable):
+    jobs: QuerySet
     name = models.CharField(max_length=100)
     description = models.TextField(default="", blank=True)
 
@@ -60,13 +65,18 @@ def user_can_edit(self, user: User) -> bool:
     def set_correct_status(self, save: bool = True):
         from api_app.models import Job
 
+        logger.info(f"Setting status for investigation {self.pk}")
         # if I have some jobs
         if self.jobs.exists():
             # and at least one is running
             for job in self.jobs.all():
                 job: Job
                 jobs = job.get_tree(job)
-                if jobs.exclude(status__in=Job.STATUSES.final_statuses()).count() > 0:
+                running_jobs = jobs.exclude(status__in=Job.STATUSES.final_statuses())
+                if running_jobs.count() > 0:
+                    logger.info(
+                        f"Jobs {running_jobs.values_list('pk', flat=True)}  are still running for investigation {self.pk}"
+                    )
                     self.status = self.STATUSES.RUNNING.value
                     self.end_time = None
                     break

api_app/investigations_manager/views.py

@@ -102,10 +102,13 @@ def remove_job(self, request, pk):
         investigation.refresh_from_db()
         # we are possibly changing the status of the investigation
         investigation.set_correct_status(save=True)
-        return Response(
+        response = Response(
             status=status.HTTP_200_OK,
             data=InvestigationSerializer(instance=investigation).data,
         )
+        if not investigation.jobs.exists():
+            investigation.delete()
+        return response
 
     @action(methods=["GET"], url_name="graph", detail=True)
     def tree(self, request, pk):

api_app/mixins.py

@@ -104,11 +104,25 @@ def _perform_request(
         self, uri: str, method: str, ignore_404: bool = False, **kwargs
     ) -> Dict:
         error = None
+        response = None
         try:
             url = self.url + uri
+            logger.debug(f"kwargs: {kwargs}")
             if method == "GET":
-                response = requests.get(url, headers=self.headers, **kwargs)
+                if kwargs:
+                    logger.debug(
+                        f"about to send get request to url {url} with headers {self.headers} and kwargs: {kwargs}"
+                    )
+                    response = requests.get(url, headers=self.headers, **kwargs)
+                else:
+                    logger.debug(
+                        f"about to send get request to url {url} with headers {self.headers} and no kwargs"
+                    )
+                    response = requests.get(url, headers=self.headers)
             elif method == "POST":
+                logger.debug(
+                    f"about to send post request to url {url} with headers {self.headers} and kwargs: {kwargs}"
+                )
                 response = requests.post(url, headers=self.headers, **kwargs)
             else:
                 raise NotImplementedError()
@@ -121,6 +135,9 @@ def _perform_request(
             # .. it happens when a requested object is not found and that's normal
             if not ignore_404 or not response.status_code == 404:
                 response.raise_for_status()
+        except requests.exceptions.JSONDecodeError as e:
+            error_message = f"Raised JSONDecodeError: {e}. Error data: {response.text if response is not None else None}"
+            raise AnalyzerRunException(error_message)
         except Exception as e:
             error_message = f"Raised Error: {e}. Error data: {error}"
             raise AnalyzerRunException(error_message)
@@ -414,7 +431,9 @@ def _vt_scan_file(self, md5: str, rescan_instead: bool = False) -> Dict:
         else:
             logger.info(f"(Job: {self.job_id}, {md5}) -> VT analyzer requested scan")
             try:
+                self._job.file.seek(0)
                 binary = self._job.file.read()
+                logger.debug(f"BINARY: {binary}")
             except Exception:
                 raise AnalyzerRunException(
                     "IntelOwl error: couldn't retrieve the binary"
@@ -444,7 +463,8 @@ def _vt_scan_file(self, md5: str, rescan_instead: bool = False) -> Dict:
         )
         for chance in range(max_tries):
             time.sleep(poll_distance)
-            result, _ = self._perform_get_request(uri, files=files)
+            result, _ = self._perform_get_request(uri)
+            logger.debug(f"result: {result}")
             analysis_status = (
                 result.get("data", {}).get("attributes", {}).get("status", "")
             )

api_app/models.py

@@ -562,14 +562,13 @@ def set_final_status(self) -> None:
                 "finished_analysis_time",
             ]
         )
-        try:
-            # we update the status of the analysis
-            if root_investigation := self.get_root().investigation:
-                root_investigation.set_correct_status(save=True)
-        except Exception as e:
-            logger.exception(
-                f"investigation status not updated. Job: {self.pk}. Error: {e}"
-            )
+        # we update the status of the analysis
+        if root_investigation := self.get_root().investigation:
+            from api_app.investigations_manager.models import Investigation
+
+            logger.info(f"Updating status of investigation {root_investigation.pk}")
+            root_investigation: Investigation
+            root_investigation.set_correct_status(save=True)
 
     def __get_config_reports(self, config: typing.Type["AbstractConfig"]) -> QuerySet:
         return getattr(self, f"{config.__name__.split('Config')[0].lower()}reports")

api_app/queryset.py

@@ -15,13 +15,13 @@
 from treebeard.mp_tree import MP_NodeQuerySet
 
 if TYPE_CHECKING:
-    from api_app.models import PythonConfig, AbstractConfig
+    from api_app.models import PythonConfig
     from api_app.serializers import AbstractBIInterface
 
 import logging
 
 from celery.canvas import Signature
-from django.db import models
+from django.db import IntegrityError, models
 from django.db.models import (
     BooleanField,
     Case,
@@ -280,7 +280,16 @@ def create(self, parent=None, **kwargs):
         """
         if parent:
             return parent.add_child(**kwargs)
-        return self.model.add_root(**kwargs)
+        try:
+            return self.model.add_root(**kwargs)
+        except IntegrityError as e:
+            if "path" in str(e):
+                logger.warning(
+                    f"Found race condition for {kwargs['name']}. Trying again to calculate path."
+                )
+                # we try again a second time, hoping for no race condition
+                return self.model.add_root(**kwargs)
+            raise
 
     def delete(self, *args, **kwargs):
         """

api_app/serializers/job.py

@@ -568,8 +568,9 @@ def get_fields(self):
             )
         return super().get_fields()
 
-    @staticmethod
-    def get_analyzers_data_model(instance: Job):
+    def get_analyzers_data_model(self, instance: Job):
+        if instance.observable_classification == ObservableTypes.GENERIC:
+            return []
         return instance.analyzerreports.get_data_models(instance).serialize()
 
 

docker/.env

@@ -1,6 +1,6 @@
 ### DO NOT CHANGE THIS VALUE !!
 ### It should be updated only when you pull latest changes off from the 'master' branch of IntelOwl.
 # this variable must start with "REACT_APP_" to be used in the frontend too
-REACT_APP_INTELOWL_VERSION=v6.2.0
+REACT_APP_INTELOWL_VERSION=v6.2.1
 # if you want to use a nfs volume for shared files
 # NFS_ADDRESS=

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "intelowl",
-  "version": "6.2.0",
+  "version": "6.2.1",
   "private": true,
   "proxy": "http://localhost:80/",
   "dependencies": {

integrations/malware_tools_analyzers/requirements/flask-requirements.txt

@@ -1,3 +1,3 @@
 Flask-Shell2HTTP-fork==1.9.2
-gunicorn==22.0.0
-fangfrisch==1.6.0
+gunicorn==23.0.0
+fangfrisch==1.9.0

integrations/malware_tools_analyzers/requirements/mobsf-requirements.txt

@@ -1,2 +1,2 @@
-mobsfscan==0.3.9
+mobsfscan==0.4.5
 

integrations/malware_tools_analyzers/requirements/thug-requirements.txt

@@ -1,4 +1,6 @@
 setuptools==70.0.0
 pytesseract==0.3.10
 pygraphviz==1.10
+# CAREFUL! This is strictly tied to STpy version and Python version
+# DO NOT UPGRADE THIS WITHOUT PROPER MANUAL TESTING
 thug==4.9

integrations/pcap_analyzers/requirements.txt

@@ -1,3 +1,3 @@
 Flask-Shell2HTTP-fork==1.9.2
-gunicorn==22.0.0
+gunicorn==23.0.0
 PyYAML==6.0

integrations/tor_analyzers/requirements.txt

@@ -1,2 +1,2 @@
 Flask-Shell2HTTP-fork==1.9.2
-gunicorn==22.0.0
+gunicorn==23.0.0

requirements/project-requirements.txt

@@ -7,7 +7,7 @@ django-filter==24.3
 django-storages==1.14
 django-celery-beat==2.7.0
 django-celery-results==2.5.0
-django-ses == 4.2.0
+django-ses == 4.3.1
 django-iam-dbauth==0.2.1
 django-prettyjson==0.4.1
 django-silk==5.3.2
@@ -35,7 +35,7 @@ whitenoise==6.8.1
 daphne==4.1.0
 channels==4.1.0
 channels-redis==4.2.0
-elasticsearch-dsl==8.15.0
+elasticsearch-dsl==8.17.0
 
 # plugins
 GitPython==3.1.41
@@ -67,7 +67,7 @@ pypssl==2.2
 pysafebrowsing==0.1.1
 PySocks==1.7.1
 py-tlsh==4.7.2
-quark-engine==24.12.1
+quark-engine==25.1.1
 speakeasy-emulator==1.5.9
 telfhash==0.9.8
 yara-python==4.5.0
@@ -86,7 +86,7 @@ knock-subdomains==7.0.1
 dotnetfile==0.2.4
 docxpy==0.8.5
 pylnk3==0.4.2
-androguard==3.4.0a1 # version >=4.x of androguard raises a dependency conflict with quark-engine==24.12.1
+androguard==3.4.0a1 # version >=4.x of androguard raises a dependency conflict with quark-engine==25.1.1
 
 # this is required because XLMMacroDeobfuscator does not pin the following packages
 pyxlsb2==0.0.8