Opt filecheck (#663)

ZePan110 · chensuyue · pre-commit-ci[bot] · web-flow · commit 61b8fa9b51eb · 2024-09-11T18:05:35.000+08:00
* update description Signed-off-by: chensuyue <suyue.chen@intel.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Optimize path-scan. Signed-off-by: ZePan110 <ze.pan@intel.com> * For test. Signed-off-by: ZePan110 <ze.pan@intel.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix bug. Signed-off-by: ZePan110 <ze.pan@intel.com> * test1 Signed-off-by: ZePan110 <ze.pan@intel.com> * test2. Signed-off-by: ZePan110 <ze.pan@intel.com> * test3 Signed-off-by: ZePan110 <ze.pan@intel.com> * Restore test file. Signed-off-by: ZePan110 <ze.pan@intel.com> * test4 Signed-off-by: ZePan110 <ze.pan@intel.com> * Fix syntax error Signed-off-by: ZePan110 <ze.pan@intel.com> * test Signed-off-by: ZePan110 <ze.pan@intel.com> * test Signed-off-by: ZePan110 <ze.pan@intel.com> * Fix issue. Signed-off-by: ZePan110 <ze.pan@intel.com> * Remove debug output Signed-off-by: ZePan110 <ze.pan@intel.com> * Restore test file and remove debug output. Signed-off-by: ZePan110 <ze.pan@intel.com> * Remove deprecated code. Signed-off-by: ZePan110 <ze.pan@intel.com> * Modify output. Signed-off-by: ZePan110 <ze.pan@intel.com> * modify output Signed-off-by: ZePan110 <ze.pan@intel.com> * Fix file path error. Signed-off-by: ZePan110 <ze.pan@intel.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix issue Signed-off-by: ZePan110 <ze.pan@intel.com> * Fix error Signed-off-by: ZePan110 <ze.pan@intel.com> * Fix error. Signed-off-by: ZePan110 <ze.pan@intel.com> * Fix error. Signed-off-by: ZePan110 <ze.pan@intel.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix error Signed-off-by: ZePan110 <ze.pan@intel.com> * resolve confilct. Signed-off-by: ZePan110 <ze.pan@intel.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix error. Signed-off-by: ZePan110 <ze.pan@intel.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: chensuyue <suyue.chen@intel.com> Signed-off-by: ZePan110 <ze.pan@intel.com> Co-authored-by: chen, suyue <suyue.chen@intel.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
diff --git a/.github/workflows/docker/compose/chathistory-compose.yaml b/.github/workflows/docker/compose/chathistory-compose.yaml
diff --git a/.github/workflows/docker/compose/dataprep-compose-cd.yaml b/.github/workflows/docker/compose/dataprep-compose-cd.yaml
@@ -7,10 +7,6 @@ services:
     build:
       dockerfile: comps/dataprep/redis/llama_index/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
-  dataprep-on-ray-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
   dataprep-milvus:
     build:
       dockerfile: comps/dataprep/milvus/langchain/Dockerfile
@@ -23,7 +19,15 @@ services:
     build:
       dockerfile: comps/dataprep/pinecone/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}
+  dataprep-redis: #need to rename
+    build:
+      dockerfile: comps/dataprep/multimodal/redis/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
   dataprep-vdms:
     build:
-      dockerfile: comps/dataprep/vdms/multimodal_langchain/docker/Dockerfile
+      dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
+  dataprep-vdmsxx: #need to rename
+    build:
+      dockerfile: comps/dataprep/vdms/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml
@@ -13,3 +13,7 @@ services:
     build:
       dockerfile: comps/dataprep/qdrant/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
+  dataprep-on-ray-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/embeddings-compose-cd.yaml b/.github/workflows/docker/compose/embeddings-compose-cd.yaml
@@ -14,3 +14,19 @@ services:
     build:
       dockerfile: comps/embeddings/tei/llama_index/Dockerfile
     image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest}
+  embedding-multimodal:
+    build:
+      dockerfile: comps/embeddings/multimodal_clip/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
+  bridgetower-embedder:
+    build:
+      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile
+    image: ${REGISTRY:-opea}/bridgetower-embedder:${TAG:-latest}
+  bridgetower-embedder-gaudi:
+    build:
+      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu
+    image: ${REGISTRY:-opea}/bridgetower-embedder-gaudi:${TAG:-latest}
+  embedding-multimodalxx: #need to rename
+    build:
+      dockerfile: comps/embeddings/multimodal/multimodal_langchain/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/feedback_management-compose-cd.yaml b/.github/workflows/docker/compose/feedback_management-compose-cd.yaml
@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  feedbackmanagement:
+    build:
+      dockerfile: comps/feedback_management/mongo/Dockerfile
+    image: ${REGISTRY:-opea}/feedbackmanagement:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/finetuning-compose-cd.yaml b/.github/workflows/docker/compose/finetuning-compose-cd.yaml
@@ -0,0 +1,13 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  finetuning:
+    build:
+      dockerfile: comps/finetuning/Dockerfile
+    image: ${REGISTRY:-opea}/finetuning:${TAG:-latest}
+  finetuning-gaudi:
+    build:
+      dockerfile: comps/finetuning/Dockerfile.intel_hpu
+    image: ${REGISTRY:-opea}/finetuning-gaudi:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/guardrails-compose-cd.yaml b/.github/workflows/docker/compose/guardrails-compose-cd.yaml
@@ -6,3 +6,7 @@ services:
     build:
       dockerfile: comps/guardrails/pii_detection/Dockerfile
     image: ${REGISTRY:-opea}/guardrails-pii-detection:${TAG:-latest}
+  guardrails-toxicity-detection:
+    build:
+      dockerfile: comps/guardrails/toxicity_detection/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-toxicity-detection:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/intent_detection-compose-cd.yaml b/.github/workflows/docker/compose/intent_detection-compose-cd.yaml
@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  llm-tgi:
+    build:
+      dockerfile: comps/intent_detection/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/knowledgegraphs-compose-cd.yaml b/.github/workflows/docker/compose/knowledgegraphs-compose-cd.yaml
@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  knowledge_graphs:
+    build:
+      dockerfile: comps/knowledgegraphs/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/knowledge_graphs:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/llms-compose-cd.yaml b/.github/workflows/docker/compose/llms-compose-cd.yaml
@@ -11,6 +11,10 @@ services:
       context: vllm-openvino
       dockerfile: Dockerfile.openvino
     image: ${REGISTRY:-opea}/vllm-openvino:${TAG:-latest}
+  llm-nativexx: #need to rename
+    build:
+      dockerfile: comps/llms/utils/lm-eval/Dockerfile
+    image: ${REGISTRY:-opea}/llm-native:${TAG:-latest}
   llm-vllm-llamaindex:
     build:
       dockerfile: comps/llms/text-generation/vllm/llama_index/Dockerfile
diff --git a/.github/workflows/docker/compose/lvms-compose-cd.yaml b/.github/workflows/docker/compose/lvms-compose-cd.yaml
@@ -17,7 +17,11 @@ services:
     build:
       dockerfile: comps/lvms/llava/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest}
-  lvm-tgi:
+  lvm-video-llama:
     build:
-      dockerfile: comps/lvms/tgi-llava/Dockerfile
-    image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest}
+      dockerfile: comps/lvms/video-llama/Dockerfile
+    image: ${REGISTRY:-opea}/lvm-video-llama:${TAG:-latest}
+  video-llama-lvm-server:
+    build:
+      dockerfile: comps/lvms/video-llama/dependency/Dockerfile
+    image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/lvms-compose.yaml b/.github/workflows/docker/compose/lvms-compose.yaml
@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  lvm-tgi:
+    build:
+      dockerfile: comps/lvms/tgi-llava/Dockerfile
+    image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/nginx-compose.yaml b/.github/workflows/docker/compose/nginx-compose.yaml
diff --git a/.github/workflows/docker/compose/prompt_registry-compose.yaml b/.github/workflows/docker/compose/prompt_registry-compose.yaml
diff --git a/.github/workflows/docker/compose/ragas-compose-cd.yaml b/.github/workflows/docker/compose/ragas-compose-cd.yaml
@@ -0,0 +1,7 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+services:
+  ragas:
+    build:
+      dockerfile: comps/ragas/tgi/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/ragas:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/reranks-compose-cd.yaml b/.github/workflows/docker/compose/reranks-compose-cd.yaml
@@ -14,3 +14,7 @@ services:
     build:
       dockerfile: comps/reranks/mosec/langchain/Dockerfile
     image: ${REGISTRY:-opea}/reranking-langchain-mosec:${TAG:-latest}
+  reranking-videoragqna:
+    build:
+      dockerfile: comps/reranks/video-rag-qna/Dockerfile
+    image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/retrievers-compose-cd.yaml b/.github/workflows/docker/compose/retrievers-compose-cd.yaml
@@ -19,3 +19,15 @@ services:
     build:
       dockerfile: comps/retrievers/redis/llama_index/Dockerfile
     image: ${REGISTRY:-opea}/retriever-redis-llamaindex:${TAG:-latest}
+  retriever-pathway:
+    build:
+      dockerfile: comps/retrievers/pathway/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/retriever-pathway:${TAG:-latest}
+  multimodal-retriever-redis:
+    build:
+      dockerfile: comps/retrievers/multimodal/redis/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/multimodal-retriever-redis:${TAG:-latest}
+  retriever-vdms:
+    build:
+      dockerfile: comps/retrievers/vdms/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/tts-compose-cd.yaml b/.github/workflows/docker/compose/tts-compose-cd.yaml
@@ -0,0 +1,10 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: reranking-tei
+services:
+  tts: #need to rename
+    build:
+      dockerfile: comps/tts/Dockerfile
+    image: ${REGISTRY:-opea}/tts:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/vectorstores-compose-cd.yaml b/.github/workflows/docker/compose/vectorstores-compose-cd.yaml
@@ -5,5 +5,5 @@
 services:
   vectorstore-pathway:
     build:
-      dockerfile: comps/vectorstores/pathway
+      dockerfile: comps/vectorstores/pathway/Dockerfile
     image: ${REGISTRY:-opea}/vectorstore-pathway:${TAG:-latest}
diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml
@@ -20,12 +20,18 @@ jobs:
       - name: Clean Up Working Directory
         run: sudo rm -rf ${{github.workspace}}/*
 
-      - name: Checkout code
+      - name: Checkout Code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Check for changed Dockerfile paths in yaml
+      - name: Clone GenAIExamples
+        run: |
+          cd ..
+          git clone https://github.com/opea-project/GenAIExamples
+
+      - name: Check Dockerfile Paths in Readme
+        if: always()
         run: |
           set -e
           shopt -s globstar
@@ -36,7 +42,7 @@ jobs:
           changed_files="$(git diff --name-status --diff-filter=DR ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/Dockerfile**' | cut -f2)"
           if [ -n "$changed_files" ]; then
             for file in $changed_files; do
-              if grep -q "$file" .github/workflows/docker/compose/*.yaml; then
+              if grep -q "$file" ./**/*.md; then
                 is_use="TRUE"
                 used_files+="$file "
               fi
@@ -46,64 +52,68 @@ jobs:
           if [[ "$is_use" == "TRUE" ]]; then
             echo "Warning: Changed Dockerfile paths:"
             echo "$used_files"
-            echo "Please modify the corresponding yaml in GenAIComps/.github/workflows/docker/compose and ask suyue.chen@intel.com for final confirmation."
+            echo "Please modify the corresponding README in GenAIComps."
             exit 1
           fi
 
-      - name: Check for changed Dockerfile paths in readme
+      - name: Check all Dockerfile mode and if it exists in yaml
         if: always()
         run: |
           set -e
           shopt -s globstar
+          no_add="FALSE"
           cd ${{github.workspace}}
-          is_use="FALSE"
-          used_files=""
-          merged_commit=$(git log -1 --format='%H')
-          changed_files="$(git diff --name-status --diff-filter=DR ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/Dockerfile**' | cut -f2)"
-          if [ -n "$changed_files" ]; then
-            for file in $changed_files; do
-              if grep -q "$file" ./**/*.md; then
-                is_use="TRUE"
-                used_files+="$file "
+          Dockerfiles=$(find ./comps -name '*Dockerfile*'|sed 's/^\.\///')
+          if [ -n "$Dockerfiles" ]; then
+            for Dockerfile in $Dockerfiles; do
+              service=$(echo "$Dockerfile" | awk -F '/' '{print $2}')
+              if grep -q "$Dockerfile" ../GenAIExamples/**/*build.yaml*; then
+                mode=""  #CI
+              else
+                mode="-cd" #CD
+              fi
+
+              yaml_file=${{github.workspace}}/.github/workflows/docker/compose/$service'-compose'$mode'.yaml'
+              if [ -e "$yaml_file" ]; then
+                # check service yaml if exist
+                if ! grep -q "$Dockerfile" "$yaml_file"; then
+                  echo "AR: Update $Dockerfile to .github/workflows/docker/compose/"$service"-compose"$mode".yaml."
+                  no_add="TRUE"
+                fi
+              else
+                echo "AR: Create a new compose file .github/workflows/docker/compose/"$service"-compose"$mode".yaml for $Dockerfile and fill it in."
+                no_add="TRUE"
               fi
             done
           fi
 
-          if [[ "$is_use" == "TRUE" ]]; then
-            echo "Warning: Changed Dockerfile paths:"
-            echo "$used_files"
-            echo "Please modify the corresponding README in GenAIComps."
+          if [[ "$no_add" == "TRUE" ]]; then
             exit 1
           fi
 
-      - name: Check new Dockerfile in compose yaml
+      - name: Check for the exist of Dockerfile in all yamls.
         if: always()
         run: |
-          set -xe
           shopt -s globstar
+          no_exist="FALSE"
           cd ${{github.workspace}}
-          no_add="FALSE"
-          merged_commit=$(git log -1 --format='%H')
-          changed_files="$(git diff --name-status --diff-filter=A ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/Dockerfile**' | cut -f2)"
-          changed_yamls="$(git diff --name-status --diff-filter=AM ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/*.yaml**' | cut -f2)"
-          if [ -n "$changed_files" ]; then
-            for file in $changed_files; do
-              service=$(echo "$file" | awk -F '/' '{print $2}')
-              if find "${{github.workspace}}/.github/workflows/docker/compose/" -name "*$service*" |grep -q .; then
-                if [ -n "$changed_files" ] && grep -q $service'-compose-cd.yaml' <<< "$changed_yamls"; then
-                  echo "The $file has been added to the ${{github.workspace}}/.github/workflows/docker/compose/"$service"-compose-cd.yaml."
-                else
-                  echo "Please check if the added $file is included in the yaml under path ${{github.workspace}}/.github/workflows/docker/compose/"$service"-compose-cd.yaml."
-                  no_add="TRUE"
+          yamls=$(find .github/workflows/docker/compose/ -name '*.yaml')
+          if [ -n "$yamls" ]; then
+            for yaml in $yamls; do
+              dockerfiles=$(grep 'dockerfile:' "$yaml"|sed 's/dockerfile: //')
+              for dockerfile in $dockerfiles; do
+                if [[ "$dockerfile" == *"comps/"* ]]; then
+                  if ! [ -e "$dockerfile" ]; then
+                    # cat "$yaml"
+                    echo "AR: The dockerfile path "$dockerfile" in "$yaml" does not exist, remove or update it."
+                    no_exist="TRUE"
+                  fi
                 fi
-              else
-                echo "Please create a new compose file named "$service"-compose-cd.yaml in ${{github.workspace}}/.github/workflows/docker/compose/ for $file and fill it in."
-                no_add="TRUE"
-              fi
+              done
             done
           fi
 
-          if [[ "$no_add" == "TRUE" ]]; then
+          if [[ "$no_exist" == "TRUE" ]]; then
             exit 1
           fi
 
