update CI to support dataprep_redis path level change (#155)

Signed-off-by: chensuyue <[email protected]>
Signed-off-by: letonghan <[email protected]>

Author: chensuyue

.github/workflows/microservice-test.yml

@@ -39,7 +39,7 @@ jobs:
           | grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true
           services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u)
           path_level_1=("asr" "tts")
-          path_level_3=("llms")
+          path_level_3=("llms_summarization" "llms_text-generation" "dataprep_redis")
           run_matrix="{\"include\":["
           for service in ${services}; do
               hardware="gaudi" # default hardware, set based on the changed files
@@ -48,7 +48,7 @@ jobs:
               else
                   vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py' | sort -u)
                   for vendor in ${vendors}; do
-                      if [[ "${path_level_3[@]}" =~ "${service}" ]]; then
+                      if [[ "${path_level_3[@]}" =~ "${service}_${vendor}" ]]; then
                           sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u)
                           for sub_vendor in ${sub_vendors}; do
                               run_matrix="${run_matrix}{\"service\":\"${service}_${vendor}_${sub_vendor}\",\"hardware\":\"${hardware}\"},"

comps/dataprep/redis/llama_index/docker/Dockerfile

@@ -2,14 +2,15 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-FROM ubuntu:22.04
+FROM python:3.11-slim
+
+ENV LANG C.UTF-8
 
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
+    build-essential \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
-    python3 \
-    python3-pip
+    vim
 
 RUN useradd -m -s /bin/bash user && \
     mkdir -p /home/user && \
@@ -19,12 +20,18 @@ USER user
 
 COPY comps /home/user/comps
 
-RUN pip install --no-cache-dir --upgrade pip && \
+RUN pip install --no-cache-dir --upgrade pip setuptools && \
     pip install --no-cache-dir -r /home/user/comps/dataprep/redis/llama_index/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
+USER root
+
+RUN mkdir -p /home/user/comps/dataprep/redis/llama_index/uploaded_files && chown -R user /home/user/comps/dataprep/redis/llama_index/uploaded_files
+
+USER user
+
 WORKDIR /home/user/comps/dataprep/redis/llama_index
 
-ENTRYPOINT ["python3", "prepare_doc_redis.py"]
+ENTRYPOINT ["python", "prepare_doc_redis.py"]
 

tests/test_dataprep_redis_llama_index.sh

@@ -16,14 +16,14 @@ function build_docker_images() {
 
 function start_service() {
     docker run -d --name="test-comps-dataprep-redis" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 6379:6379 -p 8001:8001 --ipc=host redis/redis-stack:7.2.0-v9
-    dataprep_service_port=5011
+    dataprep_service_port=5012
     REDIS_URL="redis://${ip_address}:6379"
     docker run -d --name="test-comps-dataprep-redis-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -p ${dataprep_service_port}:6007 --ipc=host opea/dataprep-redis:comps
     sleep 1m
 }
 
 function validate_microservice() {
-    dataprep_service_port=5011
+    dataprep_service_port=5012
     URL="http://${ip_address}:$dataprep_service_port/v1/dataprep"
     echo "Deep learning is a subset of machine learning that utilizes neural networks with multiple layers to analyze various levels of abstract data representations. It enables computers to identify patterns and make decisions with minimal human intervention by learning from large amounts of data." > ./dataprep_file.txt
     HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F 'files=@./dataprep_file.txt' -H 'Content-Type: multipart/form-data' "$URL")