Better broker mgmt inside entrypoints (intelowlproject#2491)

Signed-off-by: 0ssigeno <[email protected]>

Author: 0ssigeno

docker/entrypoints/celery_default.sh

@@ -14,7 +14,16 @@ else
    worker_number=8
 fi
 
-ARGUMENTS="-A intel_owl.celery worker -n worker_default --uid www-data --gid www-data --time-limit=10000 --pidfile= -c $worker_number -Ofair -Q default,broadcast,config -E --without-gossip"
+
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="default.fifo,config.fifo"
+else
+  queues="default,broadcast,config"
+fi
+
+
+ARGUMENTS="-A intel_owl.celery worker -n worker_default --uid www-data --gid www-data --time-limit=10000 --pidfile= -c $worker_number -Ofair -Q ${queues} -E --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"

docker/entrypoints/celery_ingestor.sh

@@ -4,7 +4,15 @@ until cd /opt/deploy/intel_owl
 do
     echo "Waiting for server volume..."
 done
-ARGUMENTS="-A intel_owl.celery worker -n worker_ingestor --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q ingestor,broadcast,config -E --autoscale=1,15 --without-gossip"
+
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="ingestor.fifo,config.fifo"
+else
+  queues="ingestor,broadcast,config"
+fi
+
+ARGUMENTS="-A intel_owl.celery worker -n worker_ingestor --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q ${queues} -E --autoscale=1,15 --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"

docker/entrypoints/celery_local.sh

@@ -4,8 +4,14 @@ until cd /opt/deploy/intel_owl
 do
     echo "Waiting for server volume..."
 done
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="local.fifo,config.fifo"
+else
+  queues="local,broadcast,config"
+fi
 
-ARGUMENTS="-A intel_owl.celery worker -n worker_local --uid www-data --time-limit=10000 --gid www-data --pidfile= -Ofair -Q local,broadcast,config -E --without-gossip"
+ARGUMENTS="-A intel_owl.celery worker -n worker_local --uid www-data --time-limit=10000 --gid www-data --pidfile= -Ofair -Q ${queues} -E --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"

docker/entrypoints/celery_long.sh

@@ -4,7 +4,14 @@ until cd /opt/deploy/intel_owl
 do
     echo "Waiting for server volume..."
 done
-ARGUMENTS="-A intel_owl.celery worker -n worker_long --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q long,broadcast,config -E --without-gossip"
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="long.fifo,config.fifo"
+else
+  queues="long,broadcast,config"
+fi
+
+ARGUMENTS="-A intel_owl.celery worker -n worker_long --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q ${queues} -E --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"