Refactor dataprep microservice

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -7,59 +7,3 @@ services:
     build:
       dockerfile: comps/dataprep/src/Dockerfile
     image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}
-  dataprep-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
-  dataprep-qdrant:
-    build:
-      dockerfile: comps/dataprep/qdrant/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
-  dataprep-on-ray-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
-  dataprep-multimodal-vdms:
-    build:
-      dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest}
-  dataprep-multimodal-redis:
-    build:
-      dockerfile: comps/dataprep/multimodal/redis/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-multimodal-redis:${TAG:-latest}
-  dataprep-redis-llama-index:
-    build:
-      dockerfile: comps/dataprep/redis/llama_index/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
-  dataprep-milvus:
-    build:
-      dockerfile: comps/dataprep/milvus/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest}
-  dataprep-pgvector:
-    build:
-      dockerfile: comps/dataprep/pgvector/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest}
-  dataprep-pinecone:
-    build:
-      dockerfile: comps/dataprep/pinecone/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}
-  dataprep-vdms:
-    build:
-      dockerfile: comps/dataprep/vdms/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
-  dataprep-neo4j:
-    build:
-      dockerfile: comps/dataprep/neo4j/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-neo4j:${TAG:-latest}
-  dataprep-neo4j-llamaindex:
-    build:
-      dockerfile: comps/dataprep/neo4j/llama_index/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-neo4j-llamaindex:${TAG:-latest}
-  dataprep-elasticsearch:
-    build:
-      dockerfile: comps/dataprep/elasticsearch/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-elasticsearch:${TAG:-latest}
-  dataprep-opensearch:
-    build:
-      dockerfile: comps/dataprep/opensearch/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-opensearch:${TAG:-latest}

README.md

@@ -47,10 +47,10 @@ The initially supported `Microservices` are described in the below table. More `
 | [ASR](./comps/asr/src/README.md)                  | NA                                                                             | [openai/whisper-small](https://huggingface.co/openai/whisper-small)                                     | NA                                                              | Xeon   | Audio-Speech-RecognitionS on Xeon CPU |
 | [TTS](./comps/tts/src/README.md)                  | NA                                                                             | [microsoft/speecht5_tts](https://huggingface.co/microsoft/speecht5_tts)                                 | NA                                                              | Gaudi2 | Text-To-Speech on Gaudi2              |
 | [TTS](./comps/tts/src/README.md)                  | NA                                                                             | [microsoft/speecht5_tts](https://huggingface.co/microsoft/speecht5_tts)                                 | NA                                                              | Xeon   | Text-To-Speech on Xeon CPU            |
-| [Dataprep](./comps/dataprep/src/README.md)        | [Qdrant](https://qdrant.tech/)                                                 | [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | NA                                                              | Gaudi2 | Dataprep on Gaudi2                    |
-| [Dataprep](./comps/dataprep/src/README.md)        | [Qdrant](https://qdrant.tech/)                                                 | [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | NA                                                              | Xeon   | Dataprep on Xeon CPU                  |
-| [Dataprep](./comps/dataprep/src/README.md)        | [Redis](https://redis.io/)                                                     | [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5)                                   | NA                                                              | Gaudi2 | Dataprep on Gaudi2                    |
-| [Dataprep](./comps/dataprep/src/README.md)        | [Redis](https://redis.io/)                                                     | [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5)                                   | NA                                                              | Xeon   | Dataprep on Xeon CPU                  |
+| [Dataprep](./comps/dataprep/README.md)            | [Qdrant](https://qdrant.tech/)                                                 | [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | NA                                                              | Gaudi2 | Dataprep on Gaudi2                    |
+| [Dataprep](./comps/dataprep/README.md)            | [Qdrant](https://qdrant.tech/)                                                 | [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | NA                                                              | Xeon   | Dataprep on Xeon CPU                  |
+| [Dataprep](./comps/dataprep/README.md)            | [Redis](https://redis.io/)                                                     | [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5)                                   | NA                                                              | Gaudi2 | Dataprep on Gaudi2                    |
+| [Dataprep](./comps/dataprep/README.md)            | [Redis](https://redis.io/)                                                     | [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5)                                   | NA                                                              | Xeon   | Dataprep on Xeon CPU                  |
 | [LLM](./comps/llms/src/text-generation/README.md) | [LangChain](https://www.langchain.com)/[LlamaIndex](https://www.llamaindex.ai) | [Intel/neural-chat-7b-v3-3](https://huggingface.co/Intel/neural-chat-7b-v3-3)                           | [TGI Gaudi](https://github.com/huggingface/tgi-gaudi)           | Gaudi2 | LLM on Gaudi2                         |
 | [LLM](./comps/llms/src/text-generation/README.md) | [LangChain](https://www.langchain.com)/[LlamaIndex](https://www.llamaindex.ai) | [Intel/neural-chat-7b-v3-3](https://huggingface.co/Intel/neural-chat-7b-v3-3)                           | [TGI](https://github.com/huggingface/text-generation-inference) | Xeon   | LLM on Xeon CPU                       |
 | [LLM](./comps/llms/src/text-generation/README.md) | [LangChain](https://www.langchain.com)/[LlamaIndex](https://www.llamaindex.ai) | [Intel/neural-chat-7b-v3-3](https://huggingface.co/Intel/neural-chat-7b-v3-3)                           | [Ray Serve](https://github.com/ray-project/ray)                 | Gaudi2 | LLM on Gaudi2                         |

comps/dataprep/README.md

@@ -19,28 +19,40 @@ export SUMMARIZE_IMAGE_VIA_LVM=1
 
 ## Dataprep Microservice with Redis
 
-For details, please refer to this [readme](redis/README.md)
+For details, please refer to this [readme](src/README_redis.md)
 
 ## Dataprep Microservice with Milvus
 
-For details, please refer to this [readme](milvus/langchain/README.md)
+For details, please refer to this [readme](src/README_milvus.md)
 
 ## Dataprep Microservice with Qdrant
 
-For details, please refer to this [readme](qdrant/langchain/README.md)
+For details, please refer to this [readme](src/README_qdrant.md)
 
 ## Dataprep Microservice with Pinecone
 
-For details, please refer to this [readme](pinecone/langchain/README.md)
+For details, please refer to this [readme](src/README_pinecone.md)
 
 ## Dataprep Microservice with PGVector
 
-For details, please refer to this [readme](pgvector/langchain/README.md)
+For details, please refer to this [readme](src/README_pgvector.md)
 
 ## Dataprep Microservice with VDMS
 
-For details, please refer to this [readme](vdms/README.md)
+For details, please refer to this [readme](src/README_vdms.md)
 
 ## Dataprep Microservice with Multimodal
 
-For details, please refer to this [readme](multimodal/redis/langchain/README.md)
+For details, please refer to this [readme](src/README_multimodal.md)
+
+## Dataprep Microservice with ElasticSearch
+
+For details, please refer to this [readme](src/README_elasticsearch.md)
+
+## Dataprep Microservice with OpenSearch
+
+For details, please refer to this [readme](src/README_opensearch.md)
+
+## Dataprep Microservice with neo4j
+
+For details, please refer to this [readme](src/README_neo4j_llamaindex.md)

comps/dataprep/vdms/multimodal_langchain/vdms_multimodal_langchain.yaml → comps/dataprep/deployment/docker_compose/compose_vdms_multimodal.yaml

@@ -8,9 +8,9 @@ services:
     container_name: vdms-vector-db
     ports:
       - "55555:55555"
-  dataprep-vdms:
-    image: opea/dataprep-vdms:latest
-    container_name: dataprep-vdms-server
+  dataprep-vdms-multimodal:
+    image: opea/dataprep-vdms-multimodal:latest
+    container_name: dataprep-vdms--multimodal-server
     ports:
       - "6007:6007"
     ipc: host