Shellcheck

.github/workflows/pull_request_automation.yml

@@ -57,6 +57,9 @@ jobs:
       - name: isort
         run: |
           isort . --profile black --filter-files --check-only --diff --skip configuration/ldap_config.py
+
+      - name: Perform ShellCheck Analysis
+        run: bash <(curl -s https://raw.githubusercontent.com/CICDToolbox/shellcheck/master/pipeline.sh)
 
   backend-tests:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -50,3 +50,8 @@ repos:
           - [email protected]
         args: ["--fix"]
         files: frontend/src/styles/.*(css|scss)$
+  - repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.7.2
+    hooks:
+      - id: shellcheck
+        args: ["--severity=warning"] 

api_app/analyzers_manager/repo_downloader.sh

@@ -7,29 +7,26 @@
 # Example: you may want to add a new repository. Add the clone here
 # Example: you may want to remove some of the rules available in the downloaded repositories. Remove them here.
 
-
 # This script can be disabled during development using REPO_DOWNLOADER_ENABLED=true env variable
-if [ "$REPO_DOWNLOADER_ENABLED" = "false" ]; then echo "Skipping repo_downloader.sh in DEVELOPMENT mode"; exit 0;  fi
+if [ "$REPO_DOWNLOADER_ENABLED" = "false" ]; then echo "Skipping repo_downloader.sh in DEVELOPMENT mode"; exit 0; fi
 
 # Download rules for quark-engine analyzer
-cd ~
+cd ~ || exit
 freshquark
 # this is the default directory used by Quark-Engine
 chown -R www-data:www-data ~/.quark-engine
 
 # Clone dictionaries for dnstwist analyzer
-cd /opt/deploy
+cd /opt/deploy || exit
 svn export https://github.com/elceef/dnstwist/tags/20230402/dictionaries dnstwist-dictionaries
 
 # download exiftool
 # https://exiftool.org/install.html#Unix
 mkdir exiftool_download
-cd exiftool_download
+cd exiftool_download || exit
 version=$(curl https://exiftool.org/ver.txt)
 echo "$version" >> exiftool_version.txt
 wget "https://exiftool.org/Image-ExifTool-$version.tar.gz"
 gzip -dc "Image-ExifTool-$version.tar.gz" | tar -xf -
-cd "Image-ExifTool-$version"
+cd "Image-ExifTool-$version" || exit
 chown -R www-data:www-data /opt/deploy/exiftool_download
-
-

docker/entrypoints/celery_beat.sh

@@ -10,5 +10,6 @@ then
     echo "Running celery with autoreload"
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
+  # shellcheck disable=SC2086
   /usr/local/bin/celery $ARGUMENTS
 fi

docker/entrypoints/celery_default.sh

@@ -29,5 +29,6 @@ then
     echo "Running celery with autoreload"
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
+  # shellcheck disable=SC2086
   /usr/local/bin/celery $ARGUMENTS
 fi

docker/entrypoints/celery_ingestor.sh

@@ -18,5 +18,5 @@ then
     echo "Running celery with autoreload"
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
-  /usr/local/bin/celery $ARGUMENTS
+  /usr/local/bin/celery "$ARGUMENTS"
 fi

docker/entrypoints/celery_local.sh

@@ -17,5 +17,5 @@ then
     echo "Running celery with autoreload"
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
-  /usr/local/bin/celery $ARGUMENTS
+  /usr/local/bin/celery "$ARGUMENTS"
 fi

docker/entrypoints/celery_long.sh

@@ -17,5 +17,5 @@ then
     echo "Running celery with autoreload"
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
-  /usr/local/bin/celery $ARGUMENTS
+  /usr/local/bin/celery "$ARGUMENTS"
 fi

docker/entrypoints/flower.sh

@@ -32,7 +32,7 @@ then
 fi
 
 CMD="/usr/local/bin/celery -A intel_owl.celery --broker ${BROKER_URL} flower --broker_api=${BROKER_URL_API} --max_tasks=1000 --max_workers=500"
-htpasswd -cb /opt/deploy/shared_htpasswd/.htpasswd ${FLOWER_USER} ${FLOWER_PWD}
+htpasswd -cb /opt/deploy/shared_htpasswd/.htpasswd "${FLOWER_USR}" "${FLOWER_PWD}"
 
 if [[ ${DEBUG} == "True" ]] && [[ ${DJANGO_TEST_SERVER} == "True" ]];
 then

docker/entrypoints/uwsgi.sh

@@ -24,8 +24,8 @@ fi
 # Collect static files
 python manage.py collectstatic --noinput
 echo "------------------------------"
-echo "DEBUG: " $DEBUG
-echo "DJANGO_TEST_SERVER: " $DJANGO_TEST_SERVER
+echo "DEBUG: " "$DEBUG"
+echo "DJANGO_TEST_SERVER: " "$DJANGO_TEST_SERVER"
 echo "------------------------------"
 CHANGELOG_NOTIFICATION_COMMAND='python manage.py changelog_notification .github/CHANGELOG.md INTELOWL --number-of-releases 3'
 ELASTIC_TEMPLATE_COMMAND='python manage.py elastic_templates'

docker/hooks/build

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 echo "display path"
-echo "$(pwd)"
+pwd  
 echo "display dockerfile path"
-echo $DOCKERFILE_PATH
+echo "$DOCKERFILE_PATH" 
 cd ..
 echo "current branch"
 echo "$SOURCE_BRANCH"
@@ -16,4 +16,4 @@ if [[ "$SOURCE_BRANCH" == "master" || "$SOURCE_BRANCH" == "test_arm" || "$SOURCE
 else
   echo "The branch is not master, proceeding with classic build"
   docker buildx build -f "$DOCKERFILE_PATH" -t "$IMAGE_NAME" --push .
-fi
+fi

docker/scripts/initdb.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 docker exec intelowl_uwsgi python3 manage.py makemigrations
 docker exec intelowl_uwsgi python3 manage.py migrate
 docker exec -ti intelowl_uwsgi python3 manage.py createsuperuser \

docker/scripts/install_crontab.sh

@@ -1 +1,2 @@
+#!/bin/bash
 cp cron/* /etc/cron.d

docker/scripts/install_logrotate.sh

@@ -1 +1,2 @@
+#!/bin/bash
 sudo cp -r logrotate /etc/logrotate.d

docker/scripts/manage.sh

@@ -1 +1,2 @@
-docker exec -ti intelowl_uwsgi python3 manage.py $@
+#!/bin/bash
+docker exec -ti intelowl_uwsgi python3 manage.py "$@"

docker/scripts/tail-logs.sh

@@ -1,2 +1,4 @@
-docker exec intelowl_uwsgi ls -al /var/log/intel_owl/$1
-docker exec -ti intelowl_uwsgi tail -f /var/log/intel_owl/$1
+#!/bin/bash  
+
+docker exec intelowl_uwsgi ls -al /var/log/intel_owl/"$1" 
+docker exec -ti intelowl_uwsgi tail -f /var/log/intel_owl/"$1"  

docker/scripts/watchman_install.sh

@@ -1,7 +1,7 @@
-#!/bin/bash
+#!/bin/bash  
 
 echo "WATCHMAN value is "
-echo $WATCHMAN
+echo "$WATCHMAN"
 
 # This script can be disabled during development using WATCHMAN=false env variable
 if [ "$WATCHMAN" = "false" ]; then echo "Skipping WATCHMAN installation because we are not in test mode"; exit 0;  fi
@@ -10,15 +10,15 @@ pip3 install --compile -r requirements/django-server-requirements.txt
 
 # install Watchman to enhance performance on the Django development Server
 # https://docs.djangoproject.com/en/3.2/ref/django-admin/#runserver
-cd /tmp
-wget https://github.com/facebook/watchman/releases/download/v2024.05.13.00/watchman-v2024.05.13.00-linux.zip
-unzip watchman-*-linux.zip
-cd watchman-*-linux/
+cd /tmp || exit  
+wget https://github.com/facebook/watchman/archive/refs/tags/v2024.05.13.00.zip
+unzip v2024.05.13.00.zip*
+cd watchman-2024.05.13.00* || exit
 mkdir -p /usr/local/{bin,lib} /usr/local/var/run/watchman
 cp bin/* /usr/local/bin
 cp lib/* /usr/local/lib
 chmod 755 /usr/local/bin/watchman
 chmod 2777 /usr/local/var/run/watchman
 wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb
 dpkg -i libssl1.1_1.1.0g-2ubuntu4_amd64.deb
-rm -rf watchman-*-linux*
+rm -rf v2024.05.13.00.zip*

initialize.sh

@@ -7,19 +7,19 @@ MINIMUM_DOCKER_COMPOSE_VERSION=2.3.4
 
 # Function to compare 2 semver version
 semantic_version_comp () {
-  if [[ $1 == $2 ]]; then
+  if [[ "$1" == "$2" ]]; then 
       echo "equalTo"
       return
   fi
 
   # Remove "v" prefix if present
-  ver1=$(echo $1 | sed 's/^v//')
-  ver2=$(echo $2 | sed 's/^v//')
+  ver1="${1//v/}"  # Used parameter substitution instead of sed (SC2001)
+  ver2="${2//v/}"
 
   # Convert version numbers to arrays
   local IFS=.
-  local i ver1=($ver1) ver2=($ver2)
-
+  read -ra ver1 <<< "$1"
+  read -ra ver2 <<< "$2"
   # Fill empty fields in ver1 with zeros
   for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do
       ver1[i]=0

integrations/cyberchef/hooks/build

@@ -1,9 +1,11 @@
 #!/bin/bash
 
 echo "display path"
-echo "$(pwd)"
+pwd  
+
 echo "display dockerfile path"
-echo $DOCKERFILE_PATH
+echo "$DOCKERFILE_PATH" 
+
 echo "current branch"
 echo "$SOURCE_BRANCH"
 
@@ -15,4 +17,4 @@ if [[ "$SOURCE_BRANCH" == "master" || "$SOURCE_BRANCH" == "test_arm" || "$SOURCE
 else
   echo "The branch is not master, proceeding with classic build"
   docker buildx build -f "$DOCKERFILE_PATH" -t "$IMAGE_NAME" --push .
-fi
+fi

integrations/malware_tools_analyzers/entrypoint.sh

@@ -1,28 +1,33 @@
 #!/bin/bash
-# diec analyzer variable
 export LD_LIBRARY_PATH="/opt/deploy/die/die_lin64_portable/base:$LD_LIBRARY_PATH"
 # without this makedirs the Dockerfile is not able to create new directories in volumes that already exist
-mkdir -p /var/run/clamav ${LOG_PATH} ${LOG_PATH}/clamav
-chown -R clamav:${USER} /var/lib/clamav /var/run/clamav ${LOG_PATH}
-chmod 755 /var/lib/clamav ${LOG_PATH} /var/run/clamav
-touch ${LOG_PATH}/gunicorn_access.log ${LOG_PATH}/gunicorn_errors.log ${LOG_PATH}/malware_tools_analyzers.log ${LOG_PATH}/malware_tools_analyzers_errors.log
-chown ${USER}:${USER} ${LOG_PATH}/gunicorn_access.log ${LOG_PATH}/gunicorn_errors.log ${LOG_PATH}/malware_tools_analyzers.log ${LOG_PATH}/malware_tools_analyzers_errors.log
-# clamav processes
+mkdir -p /var/run/clamav "${LOG_PATH}" "${LOG_PATH}/clamav"
+chown -R clamav:"${USER}" /var/lib/clamav /var/run/clamav "${LOG_PATH}"
+chmod 755 /var/lib/clamav "${LOG_PATH}" /var/run/clamav
+
+touch "${LOG_PATH}/gunicorn_access.log" "${LOG_PATH}/gunicorn_errors.log" \
+      "${LOG_PATH}/malware_tools_analyzers.log" "${LOG_PATH}/malware_tools_analyzers_errors.log"
+
+chown "${USER}:${USER}" "${LOG_PATH}/gunicorn_access.log" "${LOG_PATH}/gunicorn_errors.log" \
+      "${LOG_PATH}/malware_tools_analyzers.log" "${LOG_PATH}/malware_tools_analyzers_errors.log"
+
 echo "running freshclam"
 freshclam # download db for first time
 freshclam -d & # run updater in bg
 echo "running clamd"
 clamd --debug & # run daemon in bg
 # change user
 su malware_tools_analyzers-user -s /bin/bash
+
 echo "running fangfrisch"
 /opt/deploy/flask/venv/bin/fangfrisch --conf /etc/clamav/fangfrisch.conf initdb --force
 /opt/deploy/flask/venv/bin/fangfrisch --conf /etc/clamav/fangfrisch.conf refresh &
+
 echo "running gunicorn"
 # start flask server
 exec /opt/deploy/flask/venv/bin/gunicorn 'app:app' \
     --bind '0.0.0.0:4002' \
     --user malware_tools_analyzers-user \
-    --log-level ${LOG_LEVEL} \
-    --access-logfile ${LOG_PATH}/gunicorn_access.log \
-    --error-logfile ${LOG_PATH}/gunicorn_errors.log
+    --log-level "${LOG_LEVEL}" \
+    --access-logfile "${LOG_PATH}/gunicorn_access.log" \
+    --error-logfile "${LOG_PATH}/gunicorn_errors.log"

integrations/malware_tools_analyzers/hooks/build

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 echo "display path"
-echo "$(pwd)"
+pwd
 echo "display dockerfile path"
-echo $DOCKERFILE_PATH
+echo "$DOCKERFILE_PATH"
 echo "current branch"
 echo "$SOURCE_BRANCH"
 

integrations/nuclei_analyzer/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
-mkdir -p ${LOG_PATH}
-touch ${LOG_PATH}/gunicorn_access.log ${LOG_PATH}/gunicorn_errors.log
-chown -R ${USER}:${USER} ${LOG_PATH}
+mkdir -p "${LOG_PATH}"
+touch "${LOG_PATH}/gunicorn_access.log" "${LOG_PATH}/gunicorn_errors.log"
+chown -R "${USER}":"${USER}" "${LOG_PATH}"
 
 TEMPLATES_DIR="/opt/nuclei-api/nuclei-templates"
 echo "Updating Nuclei templates..."
@@ -17,5 +17,5 @@ fi
 echo "Templates downloaded successfully. Starting Flask API..."
 exec gunicorn 'app:app' \
     --bind '0.0.0.0:4008' \
-    --access-logfile ${LOG_PATH}/gunicorn_access.log \
-    --error-logfile ${LOG_PATH}/gunicorn_errors.log
+    --access-logfile "${LOG_PATH}"/gunicorn_access.log \
+    --error-logfile "${LOG_PATH}"/gunicorn_errors.log

integrations/nuclei_analyzer/hooks/build

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 echo "display path"
-echo "$(pwd)"
+pwd
 echo "display dockerfile path"
-echo $DOCKERFILE_PATH
+echo "$DOCKERFILE_PATH"
 echo "current branch"
 echo "$SOURCE_BRANCH"
 

integrations/pcap_analyzers/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
-mkdir -p ${LOG_PATH} ${LOG_PATH}/suricata
-touch ${LOG_PATH}/gunicorn_access.log ${LOG_PATH}/gunicorn_errors.log ${LOG_PATH}/suricata/suricata.log
-chown -R pcap_analyzers-user:pcap_analyzers-user ${LOG_PATH}
+mkdir -p "${LOG_PATH}" "${LOG_PATH}"/suricata
+touch "${LOG_PATH}"/gunicorn_access.log "${LOG_PATH}"/gunicorn_errors.log "${LOG_PATH}"/suricata/suricata.log
+chown -R pcap_analyzers-user:pcap_analyzers-user "${LOG_PATH}"
 su pcap_analyzers-user -s /bin/bash
 suricata-update update-sources
 suricata-update enable-source sslbl/ssl-fp-blacklist
@@ -15,8 +15,8 @@ crontab /etc/cron.d/suricata
 suricata --unix-socket=/tmp/suricata.socket &
 exec gunicorn 'app:app' \
     --bind '0.0.0.0:4004' \
-    --user ${USER} \
-    --log-level ${LOG_LEVEL} \
-    --access-logfile ${LOG_PATH}/gunicorn_access.log \
-    --error-logfile ${LOG_PATH}/gunicorn_errors.log
+    --user "${USER}" \
+    --log-level "${LOG_LEVEL}" \
+    --access-logfile "${LOG_PATH}"/gunicorn_access.log \
+    --error-logfile "${LOG_PATH}"/gunicorn_errors.log
 

integrations/pcap_analyzers/hooks/build

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 echo "display path"
-echo "$(pwd)"
+pwd
 echo "display dockerfile path"
-echo $DOCKERFILE_PATH
+echo "$DOCKERFILE_PATH"
 echo "current branch"
 echo "$SOURCE_BRANCH"
 

integrations/pcap_analyzers/update_signatures.sh

@@ -6,5 +6,5 @@ suricata-update enable-source etnetera/aggressive
 suricata-update enable-source tgreen/hunting
 suricata-update enable-source malsilo/win-malware
 suricata-update
-kill $(pidof suricata)
+kill "$(pidof suricata)"
 suricata --unix-socket=/tmp/suricata.socket &

integrations/phishing_analyzers/entrypoint.sh

@@ -7,7 +7,7 @@
 
 /usr/local/bin/gunicorn 'app:app' \
     --bind '0.0.0.0:4005' \
-    --log-level ${LOG_LEVEL} \
+    --log-level "${LOG_LEVEL}" \
     --user phishing-user \
     --group phishing-user \
     --access-logfile /var/log/intel_owl/phishing_analyzers/gunicorn_access.log \

integrations/phishing_analyzers/hooks/build

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 echo "display path"
-echo "$(pwd)"
+pwd
 echo "display dockerfile path"
-echo $DOCKERFILE_PATH
+echo "$DOCKERFILE_PATH"
 echo "current branch"
 echo "$SOURCE_BRANCH"