Refine dataprep test scripts (opea-project#1305)

* Refine dataprep Milvus CI
Signed-off-by: letonghan <[email protected]>

Author: letonghan

comps/dataprep/deployment/docker_compose/compose.yaml

@@ -4,6 +4,7 @@
 
 include:
   - ../../../third_parties/elasticsearch/deployment/docker_compose/compose.yaml
+  - ../../../third_parties/milvus/deployment/docker_compose/compose.yaml
   - ../../../third_parties/neo4j/deployment/docker_compose/compose.yaml
   - ../../../third_parties/opensearch/deployment/docker_compose/compose.yaml
   - ../../../third_parties/pgvector/deployment/docker_compose/compose.yaml
@@ -54,6 +55,12 @@ services:
     depends_on:
       tei-embedding-serving:
         condition: service_healthy
+      standalone:
+        condition: service_healthy
+      etcd:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
 
   dataprep-neo4j-llamaindex:
     image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}

tests/dataprep/test_dataprep_elasticsearch.sh

@@ -8,6 +8,7 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT=11100
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
@@ -16,7 +17,7 @@ function build_docker_images() {
     docker pull docker.elastic.co/elasticsearch/elasticsearch:8.16.0
 
     # build dataprep image for elasticsearch
-    docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
+    docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -30,7 +31,6 @@ function start_service() {
     export ELASTICSEARCH_PORT1=12300
     export ES_CONNECTION_STRING="http://${ip_address}:${ELASTICSEARCH_PORT1}"
     export INDEX_NAME="test-elasticsearch"
-    export TAG=comps
     service_name="elasticsearch-vector-db dataprep-elasticsearch"
     cd $WORKPATH
     cd comps/dataprep/deployment/docker_compose/

tests/dataprep/test_dataprep_milvus.sh

@@ -8,12 +8,14 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT=11101
+service_name="dataprep-milvus tei-embedding-serving etcd minio standalone"
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
     echo $(pwd)
     # dataprep milvus image
-    docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+    docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -23,23 +25,16 @@ function build_docker_images() {
 }
 
 function start_service() {
-    # start milvus vector db
-    cd $WORKPATH/comps/third_parties/milvus/deployment/docker_compose/
-    # wget https://raw.githubusercontent.com/milvus-io/milvus/v2.4.9/configs/milvus.yaml
-    # wget https://github.com/milvus-io/milvus/releases/download/v2.4.9/milvus-standalone-docker-compose.yml -O docker-compose.yml
-    # sed '/- \${DOCKER_VOLUME_DIRECTORY:-\.}\/volumes\/milvus:\/var\/lib\/milvus/a \ \ \ \ \ \ - \${DOCKER_VOLUME_DIRECTORY:-\.}\/milvus.yaml:\/milvus\/configs\/milvus.yaml' -i docker-compose.yml
-    docker compose up -d
-    sleep 30
-
     export host_ip=${ip_address}
     export TEI_EMBEDDER_PORT=12005
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export MILVUS_HOST=${ip_address}
     export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
     export LOGFLAG=true
-    service_name="dataprep-milvus tei-embedding-serving"
+
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
-    docker compose up ${service_name} -d
+    docker compose up ${service_name} -d > ${LOG_PATH}/start_services_with_compose.log
+
     sleep 1m
 }
 
@@ -124,12 +119,12 @@ function validate_microservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH
-    rm -rf milvus/
-    cid=$(docker ps -aq --filter "name=dataprep-milvus*")
-    if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
-    cid=$(docker ps -aq --filter "name=milvus-*")
-    if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
+    cd $WORKPATH/comps/third_parties/milvus/deployment/docker_compose/
+    docker compose -f compose.yaml down --remove-orphans
+
+    cd $WORKPATH/comps/dataprep/deployment/docker_compose
+    docker compose -f compose.yaml down  ${service_name} --remove-orphans
+
 }
 
 function main() {

tests/dataprep/test_dataprep_neo4j_on_intel_hpu.sh

@@ -9,11 +9,12 @@ LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT=11103
 LLM_ENDPOINT_PORT=10510
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build --no-cache -t opea/dataprep:comps --build-arg no_proxy=$no_proxy --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+    docker build --no-cache -t opea/dataprep:${TAG} --build-arg no_proxy=$no_proxy --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -27,7 +28,6 @@ function build_docker_images() {
 function start_service() {
     service_name="neo4j-apoc tei-embedding-serving tgi-gaudi-server dataprep-neo4j-llamaindex"
     export host_ip=${ip_address}
-    export TAG="comps"
     export NEO4J_AUTH="neo4j/neo4jtest"
     export NEO4J_URL="bolt://${ip_address}:7687"
     export NEO4J_USERNAME="neo4j"

tests/dataprep/test_dataprep_opensearch.sh

@@ -9,11 +9,12 @@ LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT="11104"
 OPENSEARCH_INITIAL_ADMIN_PASSWORD="StRoNgOpEa0)"
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+    docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -31,7 +32,6 @@ function start_service() {
     export INDEX_NAME="file-index"
     service_name="opensearch-vector-db dataprep-opensearch"
     export host_ip=${ip_address}
-    export TAG="comps"
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
     docker compose up ${service_name} -d
     sleep 1m

tests/dataprep/test_dataprep_pgvector.sh

@@ -8,6 +8,7 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT="11105"
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
@@ -16,7 +17,7 @@ function build_docker_images() {
     docker pull pgvector/pgvector:0.7.0-pg16
 
     # build dataprep image for pgvector
-    docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
+    docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -34,7 +35,6 @@ function start_service() {
 
     service_name="pgvector-db dataprep-pgvector"
     export host_ip=${ip_address}
-    export TAG="comps"
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
     docker compose up ${service_name} -d
     sleep 1m

tests/dataprep/test_dataprep_pinecone.sh

@@ -7,12 +7,13 @@ set -x
 WORKPATH=$(dirname "$PWD")
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT="11106"
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
 
     # build dataprep image for pinecone
-    docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
+    docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -27,7 +28,6 @@ function start_service() {
     export HUGGINGFACEHUB_API_TOKEN=$HF_TOKEN
 
     service_name="dataprep-pinecone"
-    export TAG="comps"
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
     docker compose up ${service_name} -d
     sleep 1m

tests/dataprep/test_dataprep_qdrant.sh

@@ -9,12 +9,13 @@ LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT="11107"
 TEI_EMBEDDER_PORT="10220"
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
 
     # dataprep qdrant image
-    docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+    docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -32,7 +33,6 @@ function start_service() {
     export COLLECTION_NAME="rag-qdrant"
     export QDRANT_HOST=$ip_address
     export QDRANT_PORT=6360
-    export TAG="comps"
     service_name="qdrant-vector-db tei-embedding-serving dataprep-qdrant"
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
     docker compose up ${service_name} -d

tests/dataprep/test_dataprep_redis.sh

@@ -9,11 +9,12 @@ LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT="11108"
 TEI_EMBEDDER_PORT="10221"
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+    docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -33,7 +34,6 @@ function start_service() {
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${TEI_EMBEDDER_PORT}"
     export INDEX_NAME="rag_redis"
-    export TAG="comps"
     service_name="redis-vector-db tei-embedding-serving dataprep-redis"
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
     docker compose up ${service_name} -d

tests/dataprep/test_dataprep_vdms.sh

@@ -8,11 +8,12 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 DATAPREP_PORT="11110"
+export TAG="comps"
 
 function build_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+    docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
 
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
@@ -30,7 +31,6 @@ function start_service() {
     export COLLECTION_NAME="test-comps"
     export QDRANT_HOST=$ip_address
     export QDRANT_PORT=$QDRANT_PORT
-    export TAG="comps"
     service_name="vdms-vector-db dataprep-vdms"
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
     docker compose up ${service_name} -d