From 1e25d6b2c6bca2327860fe3722f88d6cd319eb49 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Thu, 13 Jun 2024 20:41:28 +0800 Subject: [PATCH 01/16] Refactor Translation Example Signed-off-by: zehao-intel --- Translation/README.md | 48 +----- Translation/deprecated/README.md | 51 ++++++ .../langchain/docker/Dockerfile | 0 .../langchain/docker/build_docker.sh | 0 .../langchain/docker/requirements.txt | 0 .../docker/translation-app/prompts.py | 0 .../docker/translation-app/server.py | 0 .../serving/tgi_gaudi/Dockerfile | 0 .../serving/tgi_gaudi/build_docker.sh | 0 .../serving/tgi_gaudi/launch_tgi_service.sh | 0 .../tests/test_langchain_inference.sh | 0 Translation/docker/Dockerfile | 41 +++++ Translation/docker/gaudi/Dockerfile | 6 + Translation/docker/gaudi/README.md | 112 +++++++++++++ Translation/docker/gaudi/docker_compose.yaml | 76 +++++++++ Translation/docker/translation.py | 54 ++++++ Translation/{ => docker}/ui/docker/Dockerfile | 2 +- Translation/{ => docker}/ui/svelte/.env | 0 Translation/{ => docker}/ui/svelte/.gitignore | 0 Translation/{ => docker}/ui/svelte/.npmrc | 0 Translation/{ => docker}/ui/svelte/README.md | 4 +- .../{ => docker}/ui/svelte/package.json | 19 +-- .../{ => docker}/ui/svelte/postcss.config.cjs | 0 .../{ => docker}/ui/svelte/src/app.d.ts | 0 .../{ => docker}/ui/svelte/src/app.html | 0 .../{ => docker}/ui/svelte/src/app.pcss | 0 .../src/lib/assets/loadingAnimation.svelte | 0 .../svelte/src/lib/assets/summaryLogo.svelte | 0 .../ui/svelte/src/lib/header.svelte | 0 .../ui/svelte/src/lib/shared/Network.ts | 0 .../ui/svelte/src/lib/shared/constant.ts | 0 .../ui/svelte/src/routes/+layout.svelte | 0 .../ui/svelte/src/routes/+page.svelte | 0 .../ui/svelte/src/routes/types.d.ts | 0 .../{ => docker}/ui/svelte/static/favicon.png | Bin .../{ => docker}/ui/svelte/svelte.config.js | 0 .../ui/svelte/tailwind.config.cjs | 0 .../{ => docker}/ui/svelte/tsconfig.json | 0 .../{ => docker}/ui/svelte/vite.config.ts | 0 Translation/docker/xeon/README.md | 111 +++++++++++++ Translation/docker/xeon/docker_compose.yaml | 79 +++++++++ Translation/kubernetes/README.md | 0 Translation/kubernetes/helm-charts/README.md | 0 Translation/kubernetes/manifests/README.md | 0 Translation/kubernetes/service-mesh/README.md | 0 .../tests/test_translation_on_gaudi.sh | 157 ++++++++++++++++++ Translation/tests/test_translation_on_xeon.sh | 154 +++++++++++++++++ Translation/translation.yaml | 44 +++++ 48 files changed, 902 insertions(+), 56 deletions(-) create mode 100644 Translation/deprecated/README.md rename Translation/{ => deprecated}/langchain/docker/Dockerfile (100%) rename Translation/{ => deprecated}/langchain/docker/build_docker.sh (100%) rename Translation/{ => deprecated}/langchain/docker/requirements.txt (100%) rename Translation/{ => deprecated}/langchain/docker/translation-app/prompts.py (100%) rename Translation/{ => deprecated}/langchain/docker/translation-app/server.py (100%) rename Translation/{ => deprecated}/serving/tgi_gaudi/Dockerfile (100%) rename Translation/{ => deprecated}/serving/tgi_gaudi/build_docker.sh (100%) rename Translation/{ => deprecated}/serving/tgi_gaudi/launch_tgi_service.sh (100%) rename Translation/{ => deprecated}/tests/test_langchain_inference.sh (100%) create mode 100644 Translation/docker/Dockerfile create mode 100644 Translation/docker/gaudi/Dockerfile create mode 100644 Translation/docker/gaudi/README.md create mode 100644 Translation/docker/gaudi/docker_compose.yaml create mode 100644 Translation/docker/translation.py rename Translation/{ => docker}/ui/docker/Dockerfile (98%) rename Translation/{ => docker}/ui/svelte/.env (100%) rename Translation/{ => docker}/ui/svelte/.gitignore (100%) rename Translation/{ => docker}/ui/svelte/.npmrc (100%) rename Translation/{ => docker}/ui/svelte/README.md (82%) rename Translation/{ => docker}/ui/svelte/package.json (78%) rename Translation/{ => docker}/ui/svelte/postcss.config.cjs (100%) rename Translation/{ => docker}/ui/svelte/src/app.d.ts (100%) rename Translation/{ => docker}/ui/svelte/src/app.html (100%) rename Translation/{ => docker}/ui/svelte/src/app.pcss (100%) rename Translation/{ => docker}/ui/svelte/src/lib/assets/loadingAnimation.svelte (100%) rename Translation/{ => docker}/ui/svelte/src/lib/assets/summaryLogo.svelte (100%) rename Translation/{ => docker}/ui/svelte/src/lib/header.svelte (100%) rename Translation/{ => docker}/ui/svelte/src/lib/shared/Network.ts (100%) rename Translation/{ => docker}/ui/svelte/src/lib/shared/constant.ts (100%) rename Translation/{ => docker}/ui/svelte/src/routes/+layout.svelte (100%) rename Translation/{ => docker}/ui/svelte/src/routes/+page.svelte (100%) rename Translation/{ => docker}/ui/svelte/src/routes/types.d.ts (100%) rename Translation/{ => docker}/ui/svelte/static/favicon.png (100%) rename Translation/{ => docker}/ui/svelte/svelte.config.js (100%) rename Translation/{ => docker}/ui/svelte/tailwind.config.cjs (100%) rename Translation/{ => docker}/ui/svelte/tsconfig.json (100%) rename Translation/{ => docker}/ui/svelte/vite.config.ts (100%) create mode 100644 Translation/docker/xeon/README.md create mode 100644 Translation/docker/xeon/docker_compose.yaml create mode 100644 Translation/kubernetes/README.md create mode 100644 Translation/kubernetes/helm-charts/README.md create mode 100644 Translation/kubernetes/manifests/README.md create mode 100644 Translation/kubernetes/service-mesh/README.md create mode 100644 Translation/tests/test_translation_on_gaudi.sh create mode 100644 Translation/tests/test_translation_on_xeon.sh create mode 100644 Translation/translation.yaml diff --git a/Translation/README.md b/Translation/README.md index 3b06ad192a..546596f902 100644 --- a/Translation/README.md +++ b/Translation/README.md @@ -1,51 +1,21 @@ -# Language Translation +# Translation Application Language Translation is the communication of the meaning of a source-language text by means of an equivalent target-language text. -The workflow falls into the following architecture: +Translation architecture shows below: ![architecture](./assets/img/translation_architecture.png) -# Start Backend Service +This Translation use case performs Language Translation Inference on Intel Gaudi2 or Intel XEON Scalable Processors. The Intel Gaudi2 accelerator supports both training and inference for deep learning models in particular for LLMs. Please visit [Habana AI products](https://habana.ai/products) for more details. -1. Start the TGI Service to deploy your LLM +# Deploy Translation Service -```sh -cd serving/tgi_gaudi -bash build_docker.sh -bash launch_tgi_service.sh -``` +The Translation service can be effortlessly deployed on either Intel Gaudi2 or Intel XEON Scalable Processors. -`launch_tgi_service.sh` the script uses `8080` as the TGI service's port by default. Please replace it if any port conflicts detected. +## Deploy Translation on Gaudi -2. Start the Language Translation Service +Refer to the [Gaudi Guide](./docker/gaudi/README.md) for instructions on deploying Translation on Gaudi. -```sh -cd langchain/docker -bash build_docker.sh -docker run -it --name translation_server --net=host --ipc=host -e TGI_ENDPOINT=${TGI_ENDPOINT} -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -e SERVER_PORT=8000 -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} translation:latest bash -``` +## Deploy Translation on Xeon -**Note**: Set the following parameters before running the above command - -- `TGI_ENDPOINT`: The endpoint of your TGI service, usually equal to `:`. -- `HUGGINGFACEHUB_API_TOKEN`: Your HuggingFace hub API token, usually generated [here](https://huggingface.co/settings/tokens). -- `SERVER_PORT`: The port of the Translation service on the host. - -3. Quick Test - -```sh -curl http://localhost:8000/v1/translation \ - -X POST \ - -d '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' \ - -H 'Content-Type: application/json' -``` - -The shortcodes of languages are also supported: - -```sh -curl http://localhost:8000/v1/translation \ - -X POST \ - -d '{"language_from": "de","language_to": "en","source_language": "Maschinelles Lernen"}' \ - -H 'Content-Type: application/json' -``` +Refer to the [Xeon Guide](./docker/xeon/README.md) for instructions on deploying Translation on Xeon. diff --git a/Translation/deprecated/README.md b/Translation/deprecated/README.md new file mode 100644 index 0000000000..366866231f --- /dev/null +++ b/Translation/deprecated/README.md @@ -0,0 +1,51 @@ +# Language Translation + +Language Translation is the communication of the meaning of a source-language text by means of an equivalent target-language text. + +The workflow falls into the following architecture: + +![architecture](../assets/img/translation_architecture.png) + +# Start Backend Service + +1. Start the TGI Service to deploy your LLM + +```sh +cd serving/tgi_gaudi +bash build_docker.sh +bash launch_tgi_service.sh +``` + +`launch_tgi_service.sh` the script uses `8080` as the TGI service's port by default. Please replace it if any port conflicts detected. + +2. Start the Language Translation Service + +```sh +cd langchain/docker +bash build_docker.sh +docker run -it --name translation_server --net=host --ipc=host -e TGI_ENDPOINT=${TGI_ENDPOINT} -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -e SERVER_PORT=8000 -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} translation:latest bash +``` + +**Note**: Set the following parameters before running the above command + +- `TGI_ENDPOINT`: The endpoint of your TGI service, usually equal to `:`. +- `HUGGINGFACEHUB_API_TOKEN`: Your HuggingFace hub API token, usually generated [here](https://huggingface.co/settings/tokens). +- `SERVER_PORT`: The port of the Translation service on the host. + +3. Quick Test + +```sh +curl http://localhost:8000/v1/translation \ + -X POST \ + -d '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' \ + -H 'Content-Type: application/json' +``` + +The shortcodes of languages are also supported: + +```sh +curl http://localhost:8000/v1/translation \ + -X POST \ + -d '{"language_from": "de","language_to": "en","source_language": "Maschinelles Lernen"}' \ + -H 'Content-Type: application/json' +``` diff --git a/Translation/langchain/docker/Dockerfile b/Translation/deprecated/langchain/docker/Dockerfile similarity index 100% rename from Translation/langchain/docker/Dockerfile rename to Translation/deprecated/langchain/docker/Dockerfile diff --git a/Translation/langchain/docker/build_docker.sh b/Translation/deprecated/langchain/docker/build_docker.sh similarity index 100% rename from Translation/langchain/docker/build_docker.sh rename to Translation/deprecated/langchain/docker/build_docker.sh diff --git a/Translation/langchain/docker/requirements.txt b/Translation/deprecated/langchain/docker/requirements.txt similarity index 100% rename from Translation/langchain/docker/requirements.txt rename to Translation/deprecated/langchain/docker/requirements.txt diff --git a/Translation/langchain/docker/translation-app/prompts.py b/Translation/deprecated/langchain/docker/translation-app/prompts.py similarity index 100% rename from Translation/langchain/docker/translation-app/prompts.py rename to Translation/deprecated/langchain/docker/translation-app/prompts.py diff --git a/Translation/langchain/docker/translation-app/server.py b/Translation/deprecated/langchain/docker/translation-app/server.py similarity index 100% rename from Translation/langchain/docker/translation-app/server.py rename to Translation/deprecated/langchain/docker/translation-app/server.py diff --git a/Translation/serving/tgi_gaudi/Dockerfile b/Translation/deprecated/serving/tgi_gaudi/Dockerfile similarity index 100% rename from Translation/serving/tgi_gaudi/Dockerfile rename to Translation/deprecated/serving/tgi_gaudi/Dockerfile diff --git a/Translation/serving/tgi_gaudi/build_docker.sh b/Translation/deprecated/serving/tgi_gaudi/build_docker.sh similarity index 100% rename from Translation/serving/tgi_gaudi/build_docker.sh rename to Translation/deprecated/serving/tgi_gaudi/build_docker.sh diff --git a/Translation/serving/tgi_gaudi/launch_tgi_service.sh b/Translation/deprecated/serving/tgi_gaudi/launch_tgi_service.sh similarity index 100% rename from Translation/serving/tgi_gaudi/launch_tgi_service.sh rename to Translation/deprecated/serving/tgi_gaudi/launch_tgi_service.sh diff --git a/Translation/tests/test_langchain_inference.sh b/Translation/deprecated/tests/test_langchain_inference.sh similarity index 100% rename from Translation/tests/test_langchain_inference.sh rename to Translation/deprecated/tests/test_langchain_inference.sh diff --git a/Translation/docker/Dockerfile b/Translation/docker/Dockerfile new file mode 100644 index 0000000000..2dc37597f8 --- /dev/null +++ b/Translation/docker/Dockerfile @@ -0,0 +1,41 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FROM langchain/langchain:latest + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +RUN cd /home/user/ && \ + git clone https://github.com/opea-project/GenAIComps.git + +RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \ + pip install -r /home/user/GenAIComps/requirements.txt + +COPY ../translation.py /home/user/translation.py + +ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps + +USER user + +WORKDIR /home/user + +ENTRYPOINT ["python", "translation.py"] diff --git a/Translation/docker/gaudi/Dockerfile b/Translation/docker/gaudi/Dockerfile new file mode 100644 index 0000000000..fc1e63289e --- /dev/null +++ b/Translation/docker/gaudi/Dockerfile @@ -0,0 +1,6 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM ghcr.io/huggingface/tgi-gaudi:1.2.1 + +RUN pip install peft==0.6.2 \ No newline at end of file diff --git a/Translation/docker/gaudi/README.md b/Translation/docker/gaudi/README.md new file mode 100644 index 0000000000..ff1a536a2d --- /dev/null +++ b/Translation/docker/gaudi/README.md @@ -0,0 +1,112 @@ +# Build MegaService of Translation on Gaudi + +This document outlines the deployment process for a Translation application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as We will publish the Docker images to Docker Hub, it will simplify the deployment process for this service. + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally. This step can be ignored after the Docker images published to Docker hub. + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +``` + +### 1. Build TGI Gaudi Image + +```bash +cd GenAIExamples/Translation +docker build . -t tgi-gaudi-translation:1.2.1 --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f microservice/gaudi/Dockerfile +``` + +### 2. Build LLM Image + +```bash +cd GenAIExamples/Translation +docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile . +``` + +### 3. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `translation.py` Python script. Build the MegaService Docker image using the command below: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/Translation/microservice/gaudi/ +docker build -t opea/gen-ai-comps:translation-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +### 4. Build UI Docker Image + +Construct the frontend Docker image using the command below: + +```bash +cd GenAIExamples/Translation/ui/ +docker build -t opea/gen-ai-comps:translation-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +1. `tgi-gaudi-translation:1.2.1` +2. `opea/gen-ai-comps:llm-tgi-gaudi-server` +3. `opea/gen-ai-comps:translation-megaservice-server` +4. `opea/gen-ai-comps:translation-ui-server` + +## 🚀 Start Microservices + +### Setup Environment Variables + +Since the `docker_compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export LLM_MODEL_ID="haoranxu/ALMA-13B" +export TGI_LLM_ENDPOINT="http://${host_ip}:8008" +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export MEGA_SERVICE_HOST_IP=${host_ip} +export LLM_SERVICE_HOST_IP=${host_ip} +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/translation" +``` + +Note: Please replace with `host_ip` with you external IP address, do not use localhost. + +### Start Microservice Docker Containers + +```bash +docker compose -f docker_compose.yaml up -d +``` + +### Validate Microservices + +1. TGI Service + +```bash +curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' +``` + +2. LLM Microservice + +```bash +curl http://${host_ip}:9000/v1/chat/completions \ + -X POST \ + -d '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' \ + -H 'Content-Type: application/json' +``` + +3. MegaService + +```bash +curl http://${host_ip}:8888/v1/translation -H "Content-Type: application/json" -d '{ + "language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' +``` + +Following the validation of all aforementioned microservices, we are now prepared to construct a mega-service. + +## 🚀 Launch the UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. +![project-screenshot](https://imgur.com/yT2VDBX.png) +![project-screenshot](https://imgur.com/8ajC7lE.png) diff --git a/Translation/docker/gaudi/docker_compose.yaml b/Translation/docker/gaudi/docker_compose.yaml new file mode 100644 index 0000000000..7cd963235f --- /dev/null +++ b/Translation/docker/gaudi/docker_compose.yaml @@ -0,0 +1,76 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +services: + tgi_service: + image: tgi-gaudi-translation:1.2.1 + container_name: tgi_gaudi_service + ports: + - "8008:80" + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + volumes: + - "./data:/data" + command: --model-id ${LLM_MODEL_ID} + llm: + image: opea/llm-tgi:latest + container_name: llm-tgi-gaudi-server + depends_on: + - tgi_service + ports: + - "9000:9000" + ipc: host + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + restart: unless-stopped + translation-gaudi-backend-server: + image: opea/translation:latest + container_name: translation-gaudi-backend-server + depends_on: + - tgi_service + - llm + ports: + - "8888:8888" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + ipc: host + restart: always + translation-gaudi-ui-server: + image: opea/translation-ui:latest + container_name: translation-gaudi-ui-server + depends_on: + - translation-gaudi-backend-server + ports: + - "5173:5173" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/Translation/docker/translation.py b/Translation/docker/translation.py new file mode 100644 index 0000000000..58039cd7c2 --- /dev/null +++ b/Translation/docker/translation.py @@ -0,0 +1,54 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os + +from comps import TranslationGateway, MicroService, ServiceOrchestrator, ServiceType + +MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0") +MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 8888) +LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0") +LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000) + +class TranslationService: + def __init__(self, host="0.0.0.0", port=8000): + self.host = host + self.port = port + self.megaservice = ServiceOrchestrator() + + def add_remote_service(self): + llm = MicroService( + name="llm", + host=LLM_SERVICE_HOST_IP, + port=LLM_SERVICE_PORT, + endpoint="/v1/chat/completions", + use_remote_service=True, + service_type=ServiceType.LLM, + ) + self.megaservice.add(llm) + self.gateway = TranslationGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port) + + async def schedule(self): + await self.megaservice.schedule( + initial_inputs={"query": "Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"} + ) + result_dict = self.megaservice.result_dict + print(result_dict) + + +if __name__ == "__main__": + translation = TranslationService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT) + translation.add_remote_service() + asyncio.run(translation.schedule()) diff --git a/Translation/ui/docker/Dockerfile b/Translation/docker/ui/docker/Dockerfile similarity index 98% rename from Translation/ui/docker/Dockerfile rename to Translation/docker/ui/docker/Dockerfile index ac2bb7da31..1d5115f4b5 100644 --- a/Translation/ui/docker/Dockerfile +++ b/Translation/docker/ui/docker/Dockerfile @@ -23,4 +23,4 @@ RUN npm run build EXPOSE 5173 # Run the front-end application in preview mode -CMD ["npm", "run", "preview", "--", "--port", "5173", "--host", "0.0.0.0"] \ No newline at end of file +CMD ["npm", "run", "preview", "--", "--port", "5173", "--host", "0.0.0.0"] diff --git a/Translation/ui/svelte/.env b/Translation/docker/ui/svelte/.env similarity index 100% rename from Translation/ui/svelte/.env rename to Translation/docker/ui/svelte/.env diff --git a/Translation/ui/svelte/.gitignore b/Translation/docker/ui/svelte/.gitignore similarity index 100% rename from Translation/ui/svelte/.gitignore rename to Translation/docker/ui/svelte/.gitignore diff --git a/Translation/ui/svelte/.npmrc b/Translation/docker/ui/svelte/.npmrc similarity index 100% rename from Translation/ui/svelte/.npmrc rename to Translation/docker/ui/svelte/.npmrc diff --git a/Translation/ui/svelte/README.md b/Translation/docker/ui/svelte/README.md similarity index 82% rename from Translation/ui/svelte/README.md rename to Translation/docker/ui/svelte/README.md index 7ce136f512..3db5bf7628 100644 --- a/Translation/ui/svelte/README.md +++ b/Translation/docker/ui/svelte/README.md @@ -2,8 +2,8 @@ ### 📸 Project Screenshots -![project-screenshot](../../assets/img/trans_ui_init.png) -![project-screenshot](../../assets/img/trans_ui_select.png) +![project-screenshot](../../../assets/img/trans_ui_init.png) +![project-screenshot](../../../assets/img/trans_ui_select.png)

🧐 Features

diff --git a/Translation/ui/svelte/package.json b/Translation/docker/ui/svelte/package.json similarity index 78% rename from Translation/ui/svelte/package.json rename to Translation/docker/ui/svelte/package.json index 027dad1954..41dbb477b8 100644 --- a/Translation/ui/svelte/package.json +++ b/Translation/docker/ui/svelte/package.json @@ -3,32 +3,23 @@ "version": "0.0.1", "scripts": { "dev": "vite dev", - "build": "vite build && npm run package", + "build": "vite build", "preview": "vite preview", "package": "svelte-kit sync && svelte-package && publint", "prepublishOnly": "npm run package", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch" }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "svelte": "./dist/index.js" - } - }, - "files": [ - "dist", - "!dist/**/*.test.*", - "!dist/**/*.spec.*" - ], "peerDependencies": { "svelte": "^4.0.0" }, "devDependencies": { + "@playwright/test": "^1.44.1", "@sveltejs/adapter-auto": "^3.0.0", "@sveltejs/kit": "^2.0.0", "@sveltejs/package": "^2.0.0", "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@types/prismjs": "^1.26.3", "autoprefixer": "^10.4.16", "flowbite": "^2.3.0", "flowbite-svelte": "^0.38.5", @@ -38,15 +29,15 @@ "publint": "^0.1.9", "svelte": "^4.2.7", "svelte-check": "^3.6.0", + "svelte-highlight": "^7.6.0", "tailwindcss": "^3.3.6", "tslib": "^2.4.1", "typescript": "^5.0.0", "vite": "^5.0.11" }, - "svelte": "./dist/index.js", - "types": "./dist/index.d.ts", "type": "module", "dependencies": { + "prismjs": "^1.29.0", "sse.js": "^0.6.1", "svelte-notifications": "^0.9.98" } diff --git a/Translation/ui/svelte/postcss.config.cjs b/Translation/docker/ui/svelte/postcss.config.cjs similarity index 100% rename from Translation/ui/svelte/postcss.config.cjs rename to Translation/docker/ui/svelte/postcss.config.cjs diff --git a/Translation/ui/svelte/src/app.d.ts b/Translation/docker/ui/svelte/src/app.d.ts similarity index 100% rename from Translation/ui/svelte/src/app.d.ts rename to Translation/docker/ui/svelte/src/app.d.ts diff --git a/Translation/ui/svelte/src/app.html b/Translation/docker/ui/svelte/src/app.html similarity index 100% rename from Translation/ui/svelte/src/app.html rename to Translation/docker/ui/svelte/src/app.html diff --git a/Translation/ui/svelte/src/app.pcss b/Translation/docker/ui/svelte/src/app.pcss similarity index 100% rename from Translation/ui/svelte/src/app.pcss rename to Translation/docker/ui/svelte/src/app.pcss diff --git a/Translation/ui/svelte/src/lib/assets/loadingAnimation.svelte b/Translation/docker/ui/svelte/src/lib/assets/loadingAnimation.svelte similarity index 100% rename from Translation/ui/svelte/src/lib/assets/loadingAnimation.svelte rename to Translation/docker/ui/svelte/src/lib/assets/loadingAnimation.svelte diff --git a/Translation/ui/svelte/src/lib/assets/summaryLogo.svelte b/Translation/docker/ui/svelte/src/lib/assets/summaryLogo.svelte similarity index 100% rename from Translation/ui/svelte/src/lib/assets/summaryLogo.svelte rename to Translation/docker/ui/svelte/src/lib/assets/summaryLogo.svelte diff --git a/Translation/ui/svelte/src/lib/header.svelte b/Translation/docker/ui/svelte/src/lib/header.svelte similarity index 100% rename from Translation/ui/svelte/src/lib/header.svelte rename to Translation/docker/ui/svelte/src/lib/header.svelte diff --git a/Translation/ui/svelte/src/lib/shared/Network.ts b/Translation/docker/ui/svelte/src/lib/shared/Network.ts similarity index 100% rename from Translation/ui/svelte/src/lib/shared/Network.ts rename to Translation/docker/ui/svelte/src/lib/shared/Network.ts diff --git a/Translation/ui/svelte/src/lib/shared/constant.ts b/Translation/docker/ui/svelte/src/lib/shared/constant.ts similarity index 100% rename from Translation/ui/svelte/src/lib/shared/constant.ts rename to Translation/docker/ui/svelte/src/lib/shared/constant.ts diff --git a/Translation/ui/svelte/src/routes/+layout.svelte b/Translation/docker/ui/svelte/src/routes/+layout.svelte similarity index 100% rename from Translation/ui/svelte/src/routes/+layout.svelte rename to Translation/docker/ui/svelte/src/routes/+layout.svelte diff --git a/Translation/ui/svelte/src/routes/+page.svelte b/Translation/docker/ui/svelte/src/routes/+page.svelte similarity index 100% rename from Translation/ui/svelte/src/routes/+page.svelte rename to Translation/docker/ui/svelte/src/routes/+page.svelte diff --git a/Translation/ui/svelte/src/routes/types.d.ts b/Translation/docker/ui/svelte/src/routes/types.d.ts similarity index 100% rename from Translation/ui/svelte/src/routes/types.d.ts rename to Translation/docker/ui/svelte/src/routes/types.d.ts diff --git a/Translation/ui/svelte/static/favicon.png b/Translation/docker/ui/svelte/static/favicon.png similarity index 100% rename from Translation/ui/svelte/static/favicon.png rename to Translation/docker/ui/svelte/static/favicon.png diff --git a/Translation/ui/svelte/svelte.config.js b/Translation/docker/ui/svelte/svelte.config.js similarity index 100% rename from Translation/ui/svelte/svelte.config.js rename to Translation/docker/ui/svelte/svelte.config.js diff --git a/Translation/ui/svelte/tailwind.config.cjs b/Translation/docker/ui/svelte/tailwind.config.cjs similarity index 100% rename from Translation/ui/svelte/tailwind.config.cjs rename to Translation/docker/ui/svelte/tailwind.config.cjs diff --git a/Translation/ui/svelte/tsconfig.json b/Translation/docker/ui/svelte/tsconfig.json similarity index 100% rename from Translation/ui/svelte/tsconfig.json rename to Translation/docker/ui/svelte/tsconfig.json diff --git a/Translation/ui/svelte/vite.config.ts b/Translation/docker/ui/svelte/vite.config.ts similarity index 100% rename from Translation/ui/svelte/vite.config.ts rename to Translation/docker/ui/svelte/vite.config.ts diff --git a/Translation/docker/xeon/README.md b/Translation/docker/xeon/README.md new file mode 100644 index 0000000000..44f21307e7 --- /dev/null +++ b/Translation/docker/xeon/README.md @@ -0,0 +1,111 @@ +# Build Mega Service of Translation on Xeon + +This document outlines the deployment process for a Translation application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `llm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. + +## 🚀 Apply Xeon Server on AWS + +To apply a Xeon server on AWS, start by creating an AWS account if you don't have one already. Then, head to the [EC2 Console](https://console.aws.amazon.com/ec2/v2/home) to begin the process. Within the EC2 service, select the Amazon EC2 M7i or M7i-flex instance type to leverage the power of 4th Generation Intel Xeon Scalable processors. These instances are optimized for high-performance computing and demanding workloads. + +For detailed information about these instance types, you can refer to this [link](https://aws.amazon.com/ec2/instance-types/m7i/). Once you've chosen the appropriate instance type, proceed with configuring your instance settings, including network configurations, security groups, and storage options. + +After launching your instance, you can connect to it using SSH (for Linux instances) or Remote Desktop Protocol (RDP) (for Windows instances). From there, you'll have full access to your Xeon server, allowing you to install, configure, and manage your applications as needed. + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +``` + +### 1. Build LLM Image + +```bash +docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile . +``` + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `translation.py` Python script. Build MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/Translation/microservice/xeon/ +docker build -t opea/gen-ai-comps:translation-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +### 3. Build UI Docker Image + +Build frontend Docker image via below command: + +```bash +cd GenAIExamples/Translation/ui/ +docker build -t opea/gen-ai-comps:translation-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/gen-ai-comps:llm-tgi-server` +2. `opea/gen-ai-comps:translation-megaservice-server` +3. `opea/gen-ai-comps:translation-ui-server` + +## 🚀 Start Microservices + +### Setup Environment Variables + +Since the `docker_compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export LLM_MODEL_ID="haoranxu/ALMA-13B" +export TGI_LLM_ENDPOINT="http://${host_ip}:8008" +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export MEGA_SERVICE_HOST_IP=${host_ip} +export LLM_SERVICE_HOST_IP=${host_ip} +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/translation" +``` + +Note: Please replace with `host_ip` with you external IP address, do not use localhost. + +### Start Microservice Docker Containers + +```bash +docker compose -f docker_compose.yaml up -d +``` + +### Validate Microservices + +1. TGI Service + +```bash +curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ + -H 'Content-Type: application/json' +``` + +2. LLM Microservice + +```bash +curl http://${host_ip}:9000/v1/chat/completions \ + -X POST \ + -d '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' \ + -H 'Content-Type: application/json' +``` + +3. MegaService + +```bash +curl http://${host_ip}:8888/v1/translation -H "Content-Type: application/json" -d '{ + "language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' +``` + +Following the validation of all aforementioned microservices, we are now prepared to construct a mega-service. + +## 🚀 Launch the UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. +![project-screenshot](https://imgur.com/yT2VDBX.png) +![project-screenshot](https://imgur.com/8ajC7lE.png) diff --git a/Translation/docker/xeon/docker_compose.yaml b/Translation/docker/xeon/docker_compose.yaml new file mode 100644 index 0000000000..7e928705a9 --- /dev/null +++ b/Translation/docker/xeon/docker_compose.yaml @@ -0,0 +1,79 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +services: + tgi_service: + image: ghcr.io/huggingface/text-generation-inference:1.4 + container_name: tgi_service + ports: + - "8008:80" + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + volumes: + - "./data:/data" + shm_size: 1g + command: --model-id ${LLM_MODEL_ID} + llm: + image: opea/llm-tgi:latest + container_name: llm-tgi-server + depends_on: + - tgi_service + ports: + - "9000:9000" + ipc: host + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-llm-service" + restart: unless-stopped + translation-xeon-backend-server: + image: opea/translation:latest + container_name: translation-xeon-backend-server + depends_on: + - tgi_service + - llm + ports: + - "8888:8888" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + ipc: host + restart: always + translation-xeon-ui-server: + image: opea/translation-ui:latest + container_name: translation-xeon-ui-server + depends_on: + - translation-xeon-backend-server + ports: + - "5173:5173" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always +networks: + default: + driver: bridge diff --git a/Translation/kubernetes/README.md b/Translation/kubernetes/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/kubernetes/helm-charts/README.md b/Translation/kubernetes/helm-charts/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/kubernetes/manifests/README.md b/Translation/kubernetes/manifests/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/kubernetes/service-mesh/README.md b/Translation/kubernetes/service-mesh/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/tests/test_translation_on_gaudi.sh b/Translation/tests/test_translation_on_gaudi.sh new file mode 100644 index 0000000000..3fbd2ca49a --- /dev/null +++ b/Translation/tests/test_translation_on_gaudi.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_name=$(echo $(hostname) | tr '[a-z]-' '[A-Z]_')_$(echo 'IP') +ip_address=$(eval echo '$'$ip_name) + +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + + cd $WORKPATH/docker/gaudi + docker build . --no-cache -t tgi-gaudi-translation:1.2.1 --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . + + docker images +} + +function start_services() { + cd $WORKPATH/docker/gaudi + + export LLM_MODEL_ID="haoranxu/ALMA-13B" + export TGI_LLM_ENDPOINT="http://${ip_address}:8008" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/translation" + + # Start Docker Containers + # TODO: Replace the container name with a test-specific name + docker compose -f docker_compose.yaml up -d + + sleep 2m # Waits 2 minutes +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + + + +function validate_microservices() { + # Check if the microservices are running correctly. + # TODO: Any results check required?? + sleep 3m + # tgi gaudi service + validate_services \ + "${ip_address}:8008/generate" \ + "generated_text" \ + "tgi-gaudi" \ + "tgi_gaudi_service" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "llm-tgi-gaudi-server" \ + '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/translation" \ + "I love machine translation" \ + "mega-translation" \ + "translation-gaudi-backend-server" \ + '{"messages": "Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' +} + +function validate_frontend() { + cd $WORKPATH/docker/ui/svelte + local conda_env_name="Translation_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + conda remove -n ${conda_env_name} --all -y + conda create -n ${conda_env_name} python=3.12 -y + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs -y && npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi + +} + +function stop_docker() { + cd $WORKPATH/docker/gaudi + container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + done +} + +function main() { + + stop_docker + + build_docker_images + start_services + + validate_microservices + validate_megaservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh new file mode 100644 index 0000000000..7f218fe410 --- /dev/null +++ b/Translation/tests/test_translation_on_xeon.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_name=$(echo $(hostname) | tr '[a-z]-' '[A-Z]_')_$(echo 'IP') +ip_address=$(eval echo '$'$ip_name) + +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/translation-ui:latest -f docker/Dockerfile . + + docker images +} + +function start_services() { + cd $WORKPATH/docker/gaudi + + export LLM_MODEL_ID="haoranxu/ALMA-13B" + export TGI_LLM_ENDPOINT="http://${ip_address}:8008" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/translation" + + # Start Docker Containers + # TODO: Replace the container name with a test-specific name + docker compose -f docker_compose.yaml up -d + + sleep 2m # Waits 2 minutes +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + + + +function validate_microservices() { + # Check if the microservices are running correctly. + # TODO: Any results check required?? + sleep 3m + # tgi for llm service + validate_services \ + "${ip_address}:8008/generate" \ + "generated_text" \ + "tgi-gaudi" \ + "tgi_service" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "llm-tgi-server" \ + '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/translation" \ + "I love machine translation" \ + "mega-translation" \ + "translation-xeon-backend-server" \ + '{"messages": "Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' +} + +function validate_frontend() { + cd $WORKPATH/docker/ui/svelte + local conda_env_name="Translation_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + conda remove -n ${conda_env_name} --all -y + conda create -n ${conda_env_name} python=3.12 -y + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs -y && npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi + +} + +function stop_docker() { + cd $WORKPATH/docker/gaudi + container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + done +} + +function main() { + + stop_docker + + build_docker_images + start_services + + validate_microservices + validate_megaservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/Translation/translation.yaml b/Translation/translation.yaml new file mode 100644 index 0000000000..b27fe7b7bb --- /dev/null +++ b/Translation/translation.yaml @@ -0,0 +1,44 @@ + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +opea_micro_services: + tgi_service: + host: ${TGI_SERVICE_IP} + ports: ${TGI_SERVICE_PORT} + image: ghcr.io/huggingface/tgi-gaudi:1.2.1 + volumes: + - "./data:/data" + runtime: habana + cap_add: + - SYS_NICE + ipc: host + environment: + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + model-id: ${LLM_MODEL_ID} + llm: + host: ${LLM_SERVICE_HOST_IP} + ports: ${LLM_SERVICE_PORT} + image: opea/llm-tgi:latest + endpoint: /v1/chat/completions + environment: + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + ui: + host: ${UI_SERVICE_HOST_IP} + ports: + - "5173:5173" + image: opea/translation-ui:latest + environment: + - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + +opea_mega_service: + host: ${MEGA_SERVICE_HOST_IP} + ports: ${MEGA_SERVICE_PORT} + image: opea/translation:latest + endpoint: /v1/translation + mega_flow: + - llm From c614ba034ba0069bf30debb0f5c2917322246037 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Fri, 14 Jun 2024 10:09:28 +0800 Subject: [PATCH 02/16] support e2s test Signed-off-by: zehao-intel Signed-off-by: zehao-intel Signed-off-by: zehao-intel Signed-off-by: zehao-intel --- .github/workflows/Translation.yml | 50 ------------------- .github/workflows/scripts/build_push.sh | 4 +- Translation/docker/gaudi/Dockerfile | 6 --- Translation/docker/gaudi/README.md | 2 +- Translation/docker/gaudi/docker_compose.yaml | 2 +- Translation/docker/translation.py | 11 +--- .../tests/test_translation_on_gaudi.sh | 5 +- Translation/tests/test_translation_on_xeon.sh | 4 +- 8 files changed, 10 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/Translation.yml delete mode 100644 Translation/docker/gaudi/Dockerfile diff --git a/.github/workflows/Translation.yml b/.github/workflows/Translation.yml deleted file mode 100644 index c4b867dbc8..0000000000 --- a/.github/workflows/Translation.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -name: Translation-test - -on: - pull_request: - branches: [main] - types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - paths: - - Translation/** - - "!**.md" - - "!**/ui/**" - - .github/workflows/Translation.yml - workflow_dispatch: - -# If there is a new commit, the previous jobs will be canceled -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - Translation: - runs-on: aise-cluster - strategy: - matrix: - job_name: ["langchain"] - fail-fast: false - steps: - - name: Clean Up Working Directory - run: sudo rm -rf ${{github.workspace}}/* - - - name: Checkout out Repo - uses: actions/checkout@v4 - with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Run Test - env: - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - run: | - cd ${{ github.workspace }}/Translation/tests - bash test_${{ matrix.job_name }}_inference.sh - - - name: Publish pipeline artifact - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.job_name }} - path: ${{ github.workspace }}/Translation/tests/*.log diff --git a/.github/workflows/scripts/build_push.sh b/.github/workflows/scripts/build_push.sh index d72511cff7..31b356cb95 100755 --- a/.github/workflows/scripts/build_push.sh +++ b/.github/workflows/scripts/build_push.sh @@ -46,14 +46,14 @@ function docker_build() { # $1 is like "apple orange pear" for MEGA_SVC in $1; do case $MEGA_SVC in - "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"SearchQnA") + "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"SearchQnA"|"Translation") cd $MEGA_SVC/docker IMAGE_NAME="$(getImagenameFromMega $MEGA_SVC)" docker_build ${IMAGE_NAME} cd ui docker_build ${IMAGE_NAME}-ui docker/Dockerfile ;; - "AudioQnA"|"SearchQnA"|"Translation"|"VisualQnA") + "AudioQnA"|"SearchQnA"|"VisualQnA") echo "Not supported yet" exit 1 ;; diff --git a/Translation/docker/gaudi/Dockerfile b/Translation/docker/gaudi/Dockerfile deleted file mode 100644 index fc1e63289e..0000000000 --- a/Translation/docker/gaudi/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -FROM ghcr.io/huggingface/tgi-gaudi:1.2.1 - -RUN pip install peft==0.6.2 \ No newline at end of file diff --git a/Translation/docker/gaudi/README.md b/Translation/docker/gaudi/README.md index ff1a536a2d..e8a9524277 100644 --- a/Translation/docker/gaudi/README.md +++ b/Translation/docker/gaudi/README.md @@ -1,6 +1,6 @@ # Build MegaService of Translation on Gaudi -This document outlines the deployment process for a Translation application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as We will publish the Docker images to Docker Hub, it will simplify the deployment process for this service. +This document outlines the deployment process for a Translation application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as We will publish the Docker images to Docker Hub, it will simplify the deployment process for this service. ## 🚀 Build Docker Images diff --git a/Translation/docker/gaudi/docker_compose.yaml b/Translation/docker/gaudi/docker_compose.yaml index 7cd963235f..3656933a28 100644 --- a/Translation/docker/gaudi/docker_compose.yaml +++ b/Translation/docker/gaudi/docker_compose.yaml @@ -16,7 +16,7 @@ version: "3.8" services: tgi_service: - image: tgi-gaudi-translation:1.2.1 + image: ghcr.io/huggingface/tgi-gaudi:1.2.1 container_name: tgi_gaudi_service ports: - "8008:80" diff --git a/Translation/docker/translation.py b/Translation/docker/translation.py index 58039cd7c2..338661bfe5 100644 --- a/Translation/docker/translation.py +++ b/Translation/docker/translation.py @@ -15,13 +15,14 @@ import asyncio import os -from comps import TranslationGateway, MicroService, ServiceOrchestrator, ServiceType +from comps import MicroService, ServiceOrchestrator, ServiceType, TranslationGateway MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0") MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 8888) LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0") LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000) + class TranslationService: def __init__(self, host="0.0.0.0", port=8000): self.host = host @@ -40,15 +41,7 @@ def add_remote_service(self): self.megaservice.add(llm) self.gateway = TranslationGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port) - async def schedule(self): - await self.megaservice.schedule( - initial_inputs={"query": "Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"} - ) - result_dict = self.megaservice.result_dict - print(result_dict) - if __name__ == "__main__": translation = TranslationService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT) translation.add_remote_service() - asyncio.run(translation.schedule()) diff --git a/Translation/tests/test_translation_on_gaudi.sh b/Translation/tests/test_translation_on_gaudi.sh index 3fbd2ca49a..9c01a7fbd4 100644 --- a/Translation/tests/test_translation_on_gaudi.sh +++ b/Translation/tests/test_translation_on_gaudi.sh @@ -18,8 +18,7 @@ function build_docker_images() { cd $WORKPATH/docker docker build --no-cache -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . - cd $WORKPATH/docker/gaudi - docker build . --no-cache -t tgi-gaudi-translation:1.2.1 --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile + docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1 cd $WORKPATH/docker/ui docker build --no-cache -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . @@ -102,7 +101,7 @@ function validate_megaservice() { "I love machine translation" \ "mega-translation" \ "translation-gaudi-backend-server" \ - '{"messages": "Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' + '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' } function validate_frontend() { diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh index 7f218fe410..0367b85f2e 100644 --- a/Translation/tests/test_translation_on_xeon.sh +++ b/Translation/tests/test_translation_on_xeon.sh @@ -79,7 +79,7 @@ function validate_microservices() { validate_services \ "${ip_address}:8008/generate" \ "generated_text" \ - "tgi-gaudi" \ + "tgi" \ "tgi_service" \ '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' @@ -99,7 +99,7 @@ function validate_megaservice() { "I love machine translation" \ "mega-translation" \ "translation-xeon-backend-server" \ - '{"messages": "Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' + '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' } function validate_frontend() { From 7067ba6a65cae7f94058d78a6e81433867d8ebf2 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Wed, 19 Jun 2024 17:32:53 +0800 Subject: [PATCH 03/16] fix test ip_address Signed-off-by: zehao-intel --- Translation/tests/test_translation_on_gaudi.sh | 3 +-- Translation/tests/test_translation_on_xeon.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Translation/tests/test_translation_on_gaudi.sh b/Translation/tests/test_translation_on_gaudi.sh index 9c01a7fbd4..952c614c1d 100644 --- a/Translation/tests/test_translation_on_gaudi.sh +++ b/Translation/tests/test_translation_on_gaudi.sh @@ -6,8 +6,7 @@ set -x WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" -ip_name=$(echo $(hostname) | tr '[a-z]-' '[A-Z]_')_$(echo 'IP') -ip_address=$(eval echo '$'$ip_name) +ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { cd $WORKPATH diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh index 0367b85f2e..18da73a381 100644 --- a/Translation/tests/test_translation_on_xeon.sh +++ b/Translation/tests/test_translation_on_xeon.sh @@ -6,8 +6,7 @@ set -x WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" -ip_name=$(echo $(hostname) | tr '[a-z]-' '[A-Z]_')_$(echo 'IP') -ip_address=$(eval echo '$'$ip_name) +ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { cd $WORKPATH From 7799c318473e6cd8f6a076277299d3c1db737709 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 20 Jun 2024 10:18:09 +0800 Subject: [PATCH 04/16] update test scripts Signed-off-by: chensuyue --- .github/workflows/scripts/build_push.sh | 2 +- .../tests/test_translation_on_gaudi.sh | 24 +++++-------------- Translation/tests/test_translation_on_xeon.sh | 24 +++++-------------- 3 files changed, 13 insertions(+), 37 deletions(-) diff --git a/.github/workflows/scripts/build_push.sh b/.github/workflows/scripts/build_push.sh index 689eb068bb..ee867d4f91 100755 --- a/.github/workflows/scripts/build_push.sh +++ b/.github/workflows/scripts/build_push.sh @@ -47,7 +47,7 @@ function docker_build() { # $1 is like "apple orange pear" for MEGA_SVC in $1; do case $MEGA_SVC in - "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"SearchQnA"|"Translation") + "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"Translation") cd $MEGA_SVC/docker IMAGE_NAME="$(getImagenameFromMega $MEGA_SVC)" docker_build ${IMAGE_NAME} diff --git a/Translation/tests/test_translation_on_gaudi.sh b/Translation/tests/test_translation_on_gaudi.sh index 952c614c1d..64f54fd47a 100644 --- a/Translation/tests/test_translation_on_gaudi.sh +++ b/Translation/tests/test_translation_on_gaudi.sh @@ -8,23 +8,6 @@ WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') -function build_docker_images() { - cd $WORKPATH - git clone https://github.com/opea-project/GenAIComps.git - cd GenAIComps - docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . - - cd $WORKPATH/docker - docker build --no-cache -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . - - docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1 - - cd $WORKPATH/docker/ui - docker build --no-cache -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . - - docker images -} - function start_services() { cd $WORKPATH/docker/gaudi @@ -35,6 +18,12 @@ function start_services() { export LLM_SERVICE_HOST_IP=${ip_address} export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/translation" + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/translation:latest#image: opea/translation:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/translation-ui:latest#image: opea/translation-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + # Start Docker Containers # TODO: Replace the container name with a test-specific name docker compose -f docker_compose.yaml up -d @@ -141,7 +130,6 @@ function main() { stop_docker - build_docker_images start_services validate_microservices diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh index 18da73a381..dafbfca0eb 100644 --- a/Translation/tests/test_translation_on_xeon.sh +++ b/Translation/tests/test_translation_on_xeon.sh @@ -8,21 +8,6 @@ WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') -function build_docker_images() { - cd $WORKPATH - git clone https://github.com/opea-project/GenAIComps.git - cd GenAIComps - docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . - - cd $WORKPATH/docker - docker build --no-cache -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . - - cd $WORKPATH/docker/ui - docker build --no-cache -t opea/translation-ui:latest -f docker/Dockerfile . - - docker images -} - function start_services() { cd $WORKPATH/docker/gaudi @@ -33,6 +18,12 @@ function start_services() { export LLM_SERVICE_HOST_IP=${ip_address} export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/translation" + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/translation:latest#image: opea/translation:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/translation-ui:latest#image: opea/translation-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + # Start Docker Containers # TODO: Replace the container name with a test-specific name docker compose -f docker_compose.yaml up -d @@ -68,8 +59,6 @@ function validate_services() { sleep 1s } - - function validate_microservices() { # Check if the microservices are running correctly. # TODO: Any results check required?? @@ -139,7 +128,6 @@ function main() { stop_docker - build_docker_images start_services validate_microservices From fa5066e93a5d33b09bf9d4c9131c56db70187fd3 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 20 Jun 2024 10:27:16 +0800 Subject: [PATCH 05/16] update test scripts Signed-off-by: chensuyue --- .github/workflows/reuse-image-build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/reuse-image-build.yml b/.github/workflows/reuse-image-build.yml index 683229ef0a..01ab7c4f3b 100644 --- a/.github/workflows/reuse-image-build.yml +++ b/.github/workflows/reuse-image-build.yml @@ -34,9 +34,19 @@ jobs: image_repo: ${{ steps.build-megaservice-image.outputs.image_repo }} image_tag: ${{ steps.build-megaservice-image.outputs.image_tag }} steps: + - name: Get checkout ref + run: | + if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then + echo "CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge" >> $GITHUB_ENV + else + echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV + fi + echo "checkout ref ${{ env.CHECKOUT_REF }}" + - name: Checkout out Repo uses: actions/checkout@v4 with: + ref: ${{ env.CHECKOUT_REF }} fetch-depth: 0 - name: Building MegaService Docker Image From 34b2469c642965dfe0849a467722441e313add91 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 20 Jun 2024 10:36:15 +0800 Subject: [PATCH 06/16] for test Signed-off-by: chensuyue --- .github/workflows/docker-compose-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-compose-e2e.yml b/.github/workflows/docker-compose-e2e.yml index f8f28aca9a..e5bba60a84 100644 --- a/.github/workflows/docker-compose-e2e.yml +++ b/.github/workflows/docker-compose-e2e.yml @@ -4,7 +4,7 @@ name: E2E test with docker compose on: - pull_request_target: + pull_request: branches: [main] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: From 8cdbf9c6c5706129a750a376a0766cfed2c06a02 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Fri, 21 Jun 2024 09:58:33 +0800 Subject: [PATCH 07/16] fix readme and dockerfile Signed-off-by: zehao-intel --- Translation/docker/Dockerfile | 5 +++-- Translation/docker/gaudi/README.md | 17 +++++------------ Translation/docker/xeon/README.md | 2 +- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/Translation/docker/Dockerfile b/Translation/docker/Dockerfile index 2dc37597f8..1c4bde0a70 100644 --- a/Translation/docker/Dockerfile +++ b/Translation/docker/Dockerfile @@ -18,7 +18,8 @@ FROM langchain/langchain:latest RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ libgl1-mesa-glx \ libjemalloc-dev \ - vim + vim \ + git RUN useradd -m -s /bin/bash user && \ mkdir -p /home/user && \ @@ -30,7 +31,7 @@ RUN cd /home/user/ && \ RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \ pip install -r /home/user/GenAIComps/requirements.txt -COPY ../translation.py /home/user/translation.py +COPY ./translation.py /home/user/translation.py ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps diff --git a/Translation/docker/gaudi/README.md b/Translation/docker/gaudi/README.md index e8a9524277..ff2f09ca50 100644 --- a/Translation/docker/gaudi/README.md +++ b/Translation/docker/gaudi/README.md @@ -11,21 +11,14 @@ git clone https://github.com/opea-project/GenAIComps.git cd GenAIComps ``` -### 1. Build TGI Gaudi Image +### 1. Build LLM Image ```bash cd GenAIExamples/Translation -docker build . -t tgi-gaudi-translation:1.2.1 --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f microservice/gaudi/Dockerfile +docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . ``` -### 2. Build LLM Image - -```bash -cd GenAIExamples/Translation -docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile . -``` - -### 3. Build MegaService Docker Image +### 2. Build MegaService Docker Image To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `translation.py` Python script. Build the MegaService Docker image using the command below: @@ -35,7 +28,7 @@ cd GenAIExamples/Translation/microservice/gaudi/ docker build -t opea/gen-ai-comps:translation-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . ``` -### 4. Build UI Docker Image +### 3. Build UI Docker Image Construct the frontend Docker image using the command below: @@ -46,7 +39,7 @@ docker build -t opea/gen-ai-comps:translation-ui-server --build-arg https_proxy= Then run the command `docker images`, you will have the following four Docker Images: -1. `tgi-gaudi-translation:1.2.1` +1. `ghcr.io/huggingface/tgi-gaudi:1.2.1` 2. `opea/gen-ai-comps:llm-tgi-gaudi-server` 3. `opea/gen-ai-comps:translation-megaservice-server` 4. `opea/gen-ai-comps:translation-ui-server` diff --git a/Translation/docker/xeon/README.md b/Translation/docker/xeon/README.md index 44f21307e7..a5227b0b32 100644 --- a/Translation/docker/xeon/README.md +++ b/Translation/docker/xeon/README.md @@ -22,7 +22,7 @@ cd GenAIComps ### 1. Build LLM Image ```bash -docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile . +docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . ``` ### 2. Build MegaService Docker Image From ebf38f6c29b3b1e8f2b81cfaf676d47625f85db8 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 21 Jun 2024 12:14:03 +0800 Subject: [PATCH 08/16] revert test code Signed-off-by: chensuyue --- .github/workflows/docker-compose-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-compose-e2e.yml b/.github/workflows/docker-compose-e2e.yml index e5bba60a84..f8f28aca9a 100644 --- a/.github/workflows/docker-compose-e2e.yml +++ b/.github/workflows/docker-compose-e2e.yml @@ -4,7 +4,7 @@ name: E2E test with docker compose on: - pull_request: + pull_request_target: branches: [main] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: From 0210bd22239221ae6eb8c82239b2c321cefb21f8 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 21 Jun 2024 12:20:55 +0800 Subject: [PATCH 09/16] remove gaudi test update Signed-off-by: chensuyue --- .../tests/test_translation_on_gaudi.sh | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Translation/tests/test_translation_on_gaudi.sh b/Translation/tests/test_translation_on_gaudi.sh index 64f54fd47a..952c614c1d 100644 --- a/Translation/tests/test_translation_on_gaudi.sh +++ b/Translation/tests/test_translation_on_gaudi.sh @@ -8,6 +8,23 @@ WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + + docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1 + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . + + docker images +} + function start_services() { cd $WORKPATH/docker/gaudi @@ -18,12 +35,6 @@ function start_services() { export LLM_SERVICE_HOST_IP=${ip_address} export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/translation" - # Replace the container name with a test-specific name - echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" - sed -i "s#image: opea/translation:latest#image: opea/translation:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/translation-ui:latest#image: opea/translation-ui:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml - # Start Docker Containers # TODO: Replace the container name with a test-specific name docker compose -f docker_compose.yaml up -d @@ -130,6 +141,7 @@ function main() { stop_docker + build_docker_images start_services validate_microservices From a3c68230217111fe43795623e0bf81133356d5ba Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 21 Jun 2024 12:29:15 +0800 Subject: [PATCH 10/16] bug fix Signed-off-by: chensuyue --- Translation/tests/test_translation_on_xeon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh index dafbfca0eb..8ba67d9918 100644 --- a/Translation/tests/test_translation_on_xeon.sh +++ b/Translation/tests/test_translation_on_xeon.sh @@ -9,7 +9,7 @@ LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') function start_services() { - cd $WORKPATH/docker/gaudi + cd $WORKPATH/docker/xeon export LLM_MODEL_ID="haoranxu/ALMA-13B" export TGI_LLM_ENDPOINT="http://${ip_address}:8008" From 7d1e91f4abdf6ed6f45be749197114a0c37f2076 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Fri, 21 Jun 2024 13:20:59 +0800 Subject: [PATCH 11/16] fix test xeon Signed-off-by: zehao-intel --- Translation/tests/test_translation_on_xeon.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh index dafbfca0eb..6df300d6e0 100644 --- a/Translation/tests/test_translation_on_xeon.sh +++ b/Translation/tests/test_translation_on_xeon.sh @@ -9,7 +9,7 @@ LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') function start_services() { - cd $WORKPATH/docker/gaudi + cd $WORKPATH/docker/xeon export LLM_MODEL_ID="haoranxu/ALMA-13B" export TGI_LLM_ENDPOINT="http://${ip_address}:8008" @@ -116,7 +116,7 @@ function validate_frontend() { } function stop_docker() { - cd $WORKPATH/docker/gaudi + cd $WORKPATH/docker/xeon container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) for container_name in $container_list; do cid=$(docker ps -aq --filter "name=$container_name") From dfffde90433943d883f842f6457c2de09c5e730d Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Fri, 21 Jun 2024 14:37:56 +0800 Subject: [PATCH 12/16] modify mega check Signed-off-by: zehao-intel Signed-off-by: zehao-intel Signed-off-by: zehao-intel --- Translation/tests/test_translation_on_gaudi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Translation/tests/test_translation_on_gaudi.sh b/Translation/tests/test_translation_on_gaudi.sh index 952c614c1d..de33a4f637 100644 --- a/Translation/tests/test_translation_on_gaudi.sh +++ b/Translation/tests/test_translation_on_gaudi.sh @@ -97,7 +97,7 @@ function validate_megaservice() { # Curl the Mega Service validate_services \ "${ip_address}:8888/v1/translation" \ - "I love machine translation" \ + "translation" \ "mega-translation" \ "translation-gaudi-backend-server" \ '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' From b80d61a434c45d4daaab4d2285a65d1aef10a957 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Fri, 21 Jun 2024 17:01:12 +0800 Subject: [PATCH 13/16] fix ui Signed-off-by: zehao-intel --- Translation/docker/gaudi/docker_compose.yaml | 2 +- Translation/docker/ui/svelte/.env | 2 +- Translation/docker/xeon/docker_compose.yaml | 2 +- Translation/tests/test_translation_on_xeon.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Translation/docker/gaudi/docker_compose.yaml b/Translation/docker/gaudi/docker_compose.yaml index 3656933a28..87a044b24e 100644 --- a/Translation/docker/gaudi/docker_compose.yaml +++ b/Translation/docker/gaudi/docker_compose.yaml @@ -67,7 +67,7 @@ services: environment: - https_proxy=${https_proxy} - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + - BASE_URL=${BACKEND_SERVICE_ENDPOINT} ipc: host restart: always diff --git a/Translation/docker/ui/svelte/.env b/Translation/docker/ui/svelte/.env index 4fe9cb96ad..0a25e9a55e 100644 --- a/Translation/docker/ui/svelte/.env +++ b/Translation/docker/ui/svelte/.env @@ -1 +1 @@ -BASIC_URL = 'http://x.x.x.x:yyyy' \ No newline at end of file +BASE_URL = 'http://x.x.x.x:yyyy' \ No newline at end of file diff --git a/Translation/docker/xeon/docker_compose.yaml b/Translation/docker/xeon/docker_compose.yaml index 7e928705a9..4f627223e7 100644 --- a/Translation/docker/xeon/docker_compose.yaml +++ b/Translation/docker/xeon/docker_compose.yaml @@ -71,7 +71,7 @@ services: environment: - https_proxy=${https_proxy} - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + - BASE_URL=${BACKEND_SERVICE_ENDPOINT} ipc: host restart: always networks: diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh index 6df300d6e0..809bd1c069 100644 --- a/Translation/tests/test_translation_on_xeon.sh +++ b/Translation/tests/test_translation_on_xeon.sh @@ -84,7 +84,7 @@ function validate_megaservice() { # Curl the Mega Service validate_services \ "${ip_address}:8888/v1/translation" \ - "I love machine translation" \ + "translation" \ "mega-translation" \ "translation-xeon-backend-server" \ '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' From f1ca4f8548062a86e13809e263b24bc2850d3596 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Mon, 24 Jun 2024 16:40:32 +0800 Subject: [PATCH 14/16] fix ut network Signed-off-by: zehao-intel --- Translation/docker/gaudi/README.md | 15 +++++++-------- .../docker/ui/svelte/src/lib/shared/Network.ts | 2 +- Translation/docker/xeon/README.md | 14 +++++++------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Translation/docker/gaudi/README.md b/Translation/docker/gaudi/README.md index ff2f09ca50..b16fcaa8a1 100644 --- a/Translation/docker/gaudi/README.md +++ b/Translation/docker/gaudi/README.md @@ -14,8 +14,7 @@ cd GenAIComps ### 1. Build LLM Image ```bash -cd GenAIExamples/Translation -docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . +docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . ``` ### 2. Build MegaService Docker Image @@ -24,8 +23,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op ```bash git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/Translation/microservice/gaudi/ -docker build -t opea/gen-ai-comps:translation-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +cd GenAIExamples/Translation/docker +docker build -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . ``` ### 3. Build UI Docker Image @@ -33,8 +32,8 @@ docker build -t opea/gen-ai-comps:translation-megaservice-server --build-arg htt Construct the frontend Docker image using the command below: ```bash -cd GenAIExamples/Translation/ui/ -docker build -t opea/gen-ai-comps:translation-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +cd GenAIExamples/Translation/docker/ui/ +docker build -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . ``` Then run the command `docker images`, you will have the following four Docker Images: @@ -101,5 +100,5 @@ Following the validation of all aforementioned microservices, we are now prepare ## 🚀 Launch the UI Open this URL `http://{host_ip}:5173` in your browser to access the frontend. -![project-screenshot](https://imgur.com/yT2VDBX.png) -![project-screenshot](https://imgur.com/8ajC7lE.png) +![project-screenshot](../../assets/img/trans_ui_init.png) +![project-screenshot](../../assets/img/trans_ui_select.png) diff --git a/Translation/docker/ui/svelte/src/lib/shared/Network.ts b/Translation/docker/ui/svelte/src/lib/shared/Network.ts index f77add7959..ab505105ea 100644 --- a/Translation/docker/ui/svelte/src/lib/shared/Network.ts +++ b/Translation/docker/ui/svelte/src/lib/shared/Network.ts @@ -13,7 +13,7 @@ // limitations under the License. import { env } from "$env/dynamic/public"; -const BASIC_URL = env.BASIC_URL; +const BASIC_URL = env.BASE_URL; async function fetchPostRes(url, init) { try { diff --git a/Translation/docker/xeon/README.md b/Translation/docker/xeon/README.md index a5227b0b32..ee01e90797 100644 --- a/Translation/docker/xeon/README.md +++ b/Translation/docker/xeon/README.md @@ -22,7 +22,7 @@ cd GenAIComps ### 1. Build LLM Image ```bash -docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . +docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . ``` ### 2. Build MegaService Docker Image @@ -31,8 +31,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op ```bash git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/Translation/microservice/xeon/ -docker build -t opea/gen-ai-comps:translation-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +cd GenAIExamples/Translation/docker +docker build -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . ``` ### 3. Build UI Docker Image @@ -40,8 +40,8 @@ docker build -t opea/gen-ai-comps:translation-megaservice-server --build-arg htt Build frontend Docker image via below command: ```bash -cd GenAIExamples/Translation/ui/ -docker build -t opea/gen-ai-comps:translation-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +cd GenAIExamples/Translation/docker/ui +docker build -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . ``` Then run the command `docker images`, you will have the following Docker Images: @@ -107,5 +107,5 @@ Following the validation of all aforementioned microservices, we are now prepare ## 🚀 Launch the UI Open this URL `http://{host_ip}:5173` in your browser to access the frontend. -![project-screenshot](https://imgur.com/yT2VDBX.png) -![project-screenshot](https://imgur.com/8ajC7lE.png) +![project-screenshot](../../assets/img/trans_ui_init.png) +![project-screenshot](../../assets/img/trans_ui_select.png) From 3c080883df7a387f778e04750b342f7f02af68b1 Mon Sep 17 00:00:00 2001 From: zehao-intel Date: Mon, 24 Jun 2024 17:29:33 +0800 Subject: [PATCH 15/16] fix network Signed-off-by: zehao-intel --- Translation/docker/ui/svelte/.env | 2 +- Translation/docker/ui/svelte/src/lib/shared/Network.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Translation/docker/ui/svelte/.env b/Translation/docker/ui/svelte/.env index 0a25e9a55e..d92ea96f4b 100644 --- a/Translation/docker/ui/svelte/.env +++ b/Translation/docker/ui/svelte/.env @@ -1 +1 @@ -BASE_URL = 'http://x.x.x.x:yyyy' \ No newline at end of file +BASE_URL = 'http://backend_address:8888/v1/translation' \ No newline at end of file diff --git a/Translation/docker/ui/svelte/src/lib/shared/Network.ts b/Translation/docker/ui/svelte/src/lib/shared/Network.ts index ab505105ea..d1029c014a 100644 --- a/Translation/docker/ui/svelte/src/lib/shared/Network.ts +++ b/Translation/docker/ui/svelte/src/lib/shared/Network.ts @@ -13,7 +13,7 @@ // limitations under the License. import { env } from "$env/dynamic/public"; -const BASIC_URL = env.BASE_URL; +const BASE_URL = env.BASE_URL; async function fetchPostRes(url, init) { try { @@ -27,7 +27,7 @@ async function fetchPostRes(url, init) { } export async function fetchLanguageResponse(input: string, transform: string, transTo: string) { - const url = `${BASIC_URL}/v1/translation`; + const url = `${BASE_URL}/v1/translation`; const transData = { language_from: transform, language_to: transTo, From 20af04913b18f6f8fa6df02fb002b6948e34b252 Mon Sep 17 00:00:00 2001 From: WenjiaoYue Date: Tue, 25 Jun 2024 11:00:22 +0800 Subject: [PATCH 16/16] Modify the corresponding format according to the backend new structure. (#317) * Add image build job in docker compose e2e gaudi test in CI (#305) Signed-off-by: Yingchun Guo * Add gpu support for ChatQnA (#308) * add gpu support for chatqna Signed-off-by: Ding, Ke * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Ding, Ke Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update ChatQnA for Xeon docker_compose.yaml to fix downloads failing (#310) * Update docker_compose * Updated docker_compose * Updated docker_compose * Add build docker image option for test scripts (#312) Signed-off-by: chensuyue * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Yingchun Guo Signed-off-by: Ding, Ke Signed-off-by: chensuyue Signed-off-by: WenjiaoYue Co-authored-by: Ying Chun Guo Co-authored-by: Ke Ding Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Steve Fowler Co-authored-by: chen, suyue --- .github/workflows/docker-compose-e2e.yml | 7 +- .github/workflows/manifest-e2e.yml | 43 +-- .github/workflows/reuse-image-build.yml | 2 - .github/workflows/scripts/build_push.sh | 4 +- ChatQnA/README.md | 4 + ChatQnA/docker/gpu/README.md | 253 ++++++++++++++++++ ChatQnA/docker/gpu/docker_compose.yaml | 212 +++++++++++++++ ChatQnA/docker/xeon/docker_compose.yaml | 9 + ChatQnA/tests/test_chatqna_on_gaudi.sh | 19 +- ChatQnA/tests/test_chatqna_on_xeon.sh | 42 ++- ChatQnA/tests/test_manifest_on_gaudi.sh | 111 ++++++++ CodeGen/tests/test_codegen_on_gaudi.sh | 11 +- CodeGen/tests/test_codegen_on_xeon.sh | 30 ++- CodeGen/tests/test_manifest_on_gaudi.sh | 84 ++++++ CodeTrans/tests/test_codetrans_on_gaudi.sh | 11 +- CodeTrans/tests/test_codetrans_on_xeon.sh | 30 ++- DocSum/tests/test_docsum_on_gaudi.sh | 11 +- DocSum/tests/test_docsum_on_xeon.sh | 30 ++- README.md | 2 +- Translation/docker/ui/svelte/.env | 2 +- .../ui/svelte/src/lib/shared/Network.ts | 29 +- .../docker/ui/svelte/src/routes/+page.svelte | 47 +++- 22 files changed, 902 insertions(+), 91 deletions(-) create mode 100644 ChatQnA/docker/gpu/README.md create mode 100644 ChatQnA/docker/gpu/docker_compose.yaml create mode 100755 ChatQnA/tests/test_manifest_on_gaudi.sh create mode 100755 CodeGen/tests/test_manifest_on_gaudi.sh diff --git a/.github/workflows/docker-compose-e2e.yml b/.github/workflows/docker-compose-e2e.yml index f8f28aca9a..a0f1f42164 100644 --- a/.github/workflows/docker-compose-e2e.yml +++ b/.github/workflows/docker-compose-e2e.yml @@ -61,10 +61,13 @@ jobs: HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} example: ${{ matrix.example }} hardware: ${{ matrix.hardware }} - IMAGE_REPO: ${{needs.mega-image-build.outputs.image_repo}} - IMAGE_TAG: ${{needs.mega-image-build.outputs.image_tag}} + IMAGE_TAG: ${{ needs.mega-image-build.outputs.image_tag }} + IMAGE_REPO_GAUDI: ${{ vars.IMAGE_REPO_GAUDI }} + IMAGE_REPO_XEON: ${{ vars.IMAGE_REPO_XEON }} run: | cd ${{ github.workspace }}/$example/tests + if [ "$hardware" == "gaudi" ]; then IMAGE_REPO=$IMAGE_REPO_GAUDI; else IMAGE_REPO=$IMAGE_REPO_XEON; fi + export IMAGE_REPO=${IMAGE_REPO} example_l=$(echo $example | tr '[:upper:]' '[:lower:]') if [ -f test_${example_l}_on_${hardware}.sh ]; then timeout 30m bash test_${example_l}_on_${hardware}.sh; else echo "Test script not found, skip test!"; fi diff --git a/.github/workflows/manifest-e2e.yml b/.github/workflows/manifest-e2e.yml index 875940bd50..29e915d9f7 100644 --- a/.github/workflows/manifest-e2e.yml +++ b/.github/workflows/manifest-e2e.yml @@ -23,8 +23,8 @@ jobs: uses: ./.github/workflows/reuse-get-test-matrix.yml with: diff_excluded_files: '.github|deprecated|docker|assets|*.md|*.txt' - xeon_server_label: 'k8s' - gaudi_server_label: "" + xeon_server_label: 'xeon' + gaudi_server_label: 'gaudi' mega-image-build: needs: job1 @@ -34,12 +34,13 @@ jobs: with: image_tag: ${{ github.event.pull_request.head.sha }} mega_service: "${{ matrix.example }}" + runner_label: "docker-build-${{ matrix.hardware }}" manifest-test: needs: [job1, mega-image-build] strategy: matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }} - runs-on: ${{ matrix.hardware }} + runs-on: "k8s-${{ matrix.hardware }}" continue-on-error: true steps: - name: E2e test manifest @@ -56,35 +57,39 @@ jobs: - name: Set variables run: | - echo "IMAGE_REPO=${{needs.mega-image-build.outputs.image_repo}}" >> $GITHUB_ENV + if [ ${{ matrix.hardware }} == "gaudi" ]; then IMAGE_REPO=${{ vars.IMAGE_REPO_GAUDI }}; else IMAGE_REPO=${{ vars.IMAGE_REPO_XEON }}; fi + echo "IMAGE_REPO=${IMAGE_REPO}" >> $GITHUB_ENV echo "IMAGE_TAG=${{needs.mega-image-build.outputs.image_tag}}" >> $GITHUB_ENV lower_example=$(echo "${{ matrix.example }}" | tr '[:upper:]' '[:lower:]') echo "NAMESPACE=$lower_example-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV + echo "continue_test=true" >> $GITHUB_ENV echo "should_cleanup=false" >> $GITHUB_ENV echo "skip_validate=true" >> $GITHUB_ENV echo "NAMESPACE=$NAMESPACE" - - name: Initialize manifest testing - run: | - ${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_xeon.sh init_${{ matrix.example }} - - name: Kubectl install id: install run: | - echo "should_cleanup=true" >> $GITHUB_ENV - kubectl create ns $NAMESPACE - ${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_xeon.sh install_${{ matrix.example }} $NAMESPACE - echo "Testing ${{ matrix.example }}, waiting for pod ready..." - if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then - echo "Testing manifests ${{ matrix.example }}, waiting for pod ready done!" - echo "skip_validate=false" >> $GITHUB_ENV + if [[ ! -f ${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_${{ matrix.hardware }}.sh ]]; then + echo "No test script found, exist test!" + exit 0 else - echo "Timeout waiting for pods in namespace $NAMESPACE to be ready!" - exit 1 + ${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_${{ matrix.hardware }}.sh init_${{ matrix.example }} + echo "should_cleanup=true" >> $GITHUB_ENV + kubectl create ns $NAMESPACE + ${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_${{ matrix.hardware }}.sh install_${{ matrix.example }} $NAMESPACE + echo "Testing ${{ matrix.example }}, waiting for pod ready..." + if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then + echo "Testing manifests ${{ matrix.example }}, waiting for pod ready done!" + echo "skip_validate=false" >> $GITHUB_ENV + else + echo "Timeout waiting for pods in namespace $NAMESPACE to be ready!" + exit 1 + fi + sleep 60 fi - sleep 60 - name: Validate e2e test if: always() @@ -92,7 +97,7 @@ jobs: if $skip_validate; then echo "Skip validate" else - ${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_xeon.sh validate_${{ matrix.example }} $NAMESPACE + ${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_${{ matrix.hardware }}.sh validate_${{ matrix.example }} $NAMESPACE fi - name: Kubectl uninstall diff --git a/.github/workflows/reuse-image-build.yml b/.github/workflows/reuse-image-build.yml index 01ab7c4f3b..e04c4800cb 100644 --- a/.github/workflows/reuse-image-build.yml +++ b/.github/workflows/reuse-image-build.yml @@ -60,7 +60,5 @@ jobs: if [ -z "${{ env.IMAGE_REPO }}" ]; then IMAGE_REPO=$OPEA_IMAGE_REPO fi - echo "IMAGE_REPO=${IMAGE_REPO}" echo "IMAGE_TAG=${IMAGE_TAG}" - echo "image_repo=$IMAGE_REPO" >> $GITHUB_OUTPUT echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/build_push.sh b/.github/workflows/scripts/build_push.sh index ee867d4f91..89246006fd 100755 --- a/.github/workflows/scripts/build_push.sh +++ b/.github/workflows/scripts/build_push.sh @@ -35,8 +35,7 @@ function docker_build() { echo "Building ${IMAGE_REPO}opea/$1:$IMAGE_TAG using Dockerfile $DOCKERFILE_PATH" # if https_proxy and http_proxy are set, pass them to docker build if [ -z "$https_proxy" ]; then - #docker build --no-cache -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG -f $DOCKERFILE_PATH . - docker build -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG -f $DOCKERFILE_PATH . + docker build --no-cache -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG -f $DOCKERFILE_PATH . else docker build --no-cache -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $DOCKERFILE_PATH . fi @@ -56,7 +55,6 @@ for MEGA_SVC in $1; do ;; "AudioQnA"|"SearchQnA"|"VisualQnA") echo "Not supported yet" - exit 1 ;; *) echo "Unknown function: $MEGA_SVC" diff --git a/ChatQnA/README.md b/ChatQnA/README.md index 740ea890c4..42cac30b12 100644 --- a/ChatQnA/README.md +++ b/ChatQnA/README.md @@ -22,6 +22,10 @@ Refer to the [Gaudi Guide](./docker/gaudi/README.md) for instructions on deployi Refer to the [Xeon Guide](./docker/xeon/README.md) for instructions on deploying ChatQnA on Xeon. +## Deploy ChatQnA on NVIDIA GPU + +Refer to the [NVIDIA GPU Guide](./docker/gpu/README.md) for instructions on deploying ChatQnA on NVIDIA GPU. + ## Deploy ChatQnA into Kubernetes on Xeon & Gaudi Refer to the [Kubernetes Guide](./kubernetes/manifests/README.md) for instructions on deploying ChatQnA into Kubernetes on Xeon & Gaudi. diff --git a/ChatQnA/docker/gpu/README.md b/ChatQnA/docker/gpu/README.md new file mode 100644 index 0000000000..18b3e95721 --- /dev/null +++ b/ChatQnA/docker/gpu/README.md @@ -0,0 +1,253 @@ +# Build MegaService of ChatQnA on NVIDIA GPU + +This document outlines the deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on NVIDIA GPU platform. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as embedding, retriever, rerank, and llm. We will publish the Docker images to Docker Hub, it will simplify the deployment process for this service. + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally. This step can be ignored after the Docker images published to Docker hub. + +### 1. Source Code install GenAIComps + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +``` + +### 2. Build Embedding Image + +```bash +docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/langchain/docker/Dockerfile . +``` + +### 3. Build Retriever Image + +```bash +docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/redis/docker/Dockerfile . +``` + +### 4. Build Rerank Image + +```bash +docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/langchain/docker/Dockerfile . +``` + +### 5. Build LLM Image + +```bash +docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . +``` + +### 6. Build Dataprep Image + +```bash +docker build --no-cache -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/langchain/docker/Dockerfile . +``` + +### 7. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `chatqna.py` Python script. Build the MegaService Docker image using the command below: + +```bash +git clone https://github.com/opea-project/GenAIExamples.git +cd GenAIExamples/ChatQnA/docker +docker build --no-cache -t opea/chatqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +cd ../../.. +``` + +### 8. Build UI Docker Image + +Construct the frontend Docker image using the command below: + +```bash +cd GenAIExamples/ChatQnA/docker/ui/ +docker build --no-cache -t opea/chatqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +cd ../../../.. +``` + +Then run the command `docker images`, you will have the following 7 Docker Images: + +1. `opea/embedding-tei:latest` +2. `opea/retriever-redis:latest` +3. `opea/reranking-tei:latest` +4. `opea/llm-tgi:latest` +5. `opea/dataprep-redis:latest` +6. `opea/chatqna:latest` +7. `opea/chatqna-ui:latest` + +## 🚀 Start MicroServices and MegaService + +### Setup Environment Variables + +Since the `docker_compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export no_proxy=${your_no_proxy} +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" +export RERANK_MODEL_ID="BAAI/bge-reranker-base" +export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:8090" +export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808" +export TGI_LLM_ENDPOINT="http://${host_ip}:8008" +export REDIS_URL="redis://${host_ip}:6379" +export INDEX_NAME="rag-redis" +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export MEGA_SERVICE_HOST_IP=${host_ip} +export EMBEDDING_SERVICE_HOST_IP=${host_ip} +export RETRIEVER_SERVICE_HOST_IP=${host_ip} +export RERANK_SERVICE_HOST_IP=${host_ip} +export LLM_SERVICE_HOST_IP=${host_ip} +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/chatqna" +export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep" +``` + +Note: Please replace with `host_ip` with you external IP address, do **NOT** use localhost. + +### Start all the services Docker Containers + +```bash +cd GenAIExamples/ChatQnA/docker/gpu/ +docker compose -f docker_compose.yaml up -d +``` + +### Validate MicroServices and MegaService + +1. TEI Embedding Service + +```bash +curl ${host_ip}:8090/embed \ + -X POST \ + -d '{"inputs":"What is Deep Learning?"}' \ + -H 'Content-Type: application/json' +``` + +2. Embedding Microservice + +```bash +curl http://${host_ip}:6000/v1/embeddings \ + -X POST \ + -d '{"text":"hello"}' \ + -H 'Content-Type: application/json' +``` + +3. Retriever Microservice + +To consume the retriever microservice, you need to generate a mock embedding vector of length 768 in Python script: + +```python +import random + +embedding = [random.uniform(-1, 1) for _ in range(768)] +print(embedding) +``` + +Then substitute your mock embedding vector for the `${your_embedding}` in the following `curl` command: + +```bash +curl http://${host_ip}:7000/v1/retrieval \ + -X POST \ + -d '{"text":"test", "embedding":${your_embedding}}' \ + -H 'Content-Type: application/json' +``` + +4. TEI Reranking Service + +```bash +curl http://${host_ip}:8808/rerank \ + -X POST \ + -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \ + -H 'Content-Type: application/json' +``` + +5. Reranking Microservice + +```bash +curl http://${host_ip}:8000/v1/reranking \ + -X POST \ + -d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \ + -H 'Content-Type: application/json' +``` + +6. TGI Service + +```bash +curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' +``` + +7. LLM Microservice + +```bash +curl http://${host_ip}:9000/v1/chat/completions \ + -X POST \ + -d '{"query":"What is Deep Learning?","max_new_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"repetition_penalty":1.03,"streaming":true}' \ + -H 'Content-Type: application/json' +``` + +8. MegaService + +```bash +curl http://${host_ip}:8888/v1/chatqna -H "Content-Type: application/json" -d '{ + "messages": "What is the revenue of Nike in 2023?" + }' +``` + +9. Dataprep Microservice（Optional） + +If you want to update the default knowledge base, you can use the following commands: + +Update Knowledge Base via Local File Upload: + +```bash +curl -X POST "http://${host_ip}:6007/v1/dataprep" \ + -H "Content-Type: multipart/form-data" \ + -F "files=@./nke-10k-2023.pdf" +``` + +This command updates a knowledge base by uploading a local file for processing. Update the file path according to your environment. + +Add Knowledge Base via HTTP Links: + +```bash +curl -X POST "http://${host_ip}:6007/v1/dataprep" \ + -H "Content-Type: multipart/form-data" \ + -F 'link_list=["https://opea.dev"]' +``` + +This command updates a knowledge base by submitting a list of HTTP links for processing. + +## Enable LangSmith for Monotoring Application (Optional) + +LangSmith offers tools to debug, evaluate, and monitor language models and intelligent agents. It can be used to assess benchmark data for each microservice. Before launching your services with `docker compose -f docker_compose.yaml up -d`, you need to enable LangSmith tracing by setting the `LANGCHAIN_TRACING_V2` environment variable to true and configuring your LangChain API key. + +Here's how you can do it: + +1. Install the latest version of LangSmith: + +```bash +pip install -U langsmith +``` + +2. Set the necessary environment variables: + +```bash +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=ls_... +``` + +## 🚀 Launch the UI + +To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `docker_compose.yaml` file as shown below: + +```yaml + chaqna-ui-server: + image: opea/chatqna-ui:latest + ... + ports: + - "80:5173" +``` + +![project-screenshot](../../assets/img/chat_ui_init.png) diff --git a/ChatQnA/docker/gpu/docker_compose.yaml b/ChatQnA/docker/gpu/docker_compose.yaml new file mode 100644 index 0000000000..e19fcdb86a --- /dev/null +++ b/ChatQnA/docker/gpu/docker_compose.yaml @@ -0,0 +1,212 @@ + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3.8" + +services: + redis-vector-db: + image: redis/redis-stack:7.2.0-v9 + container_name: redis-vector-db + ports: + - "6379:6379" + - "8001:8001" + dataprep-redis-service: + image: opea/dataprep-redis:latest + container_name: dataprep-redis-server + depends_on: + - redis-vector-db + ports: + - "6007:6007" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${REDIS_URL} + INDEX_NAME: ${INDEX_NAME} + tei-embedding-service: + image: ghcr.io/huggingface/text-embeddings-inference:1.2 + container_name: tei-embedding-server + ports: + - "8090:80" + volumes: + - "./data:/data" + shm_size: 1g + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + command: --model-id ${EMBEDDING_MODEL_ID} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + embedding: + image: opea/embedding-tei:latest + container_name: embedding-tei-server + depends_on: + - tei-embedding-service + ports: + - "6000:6000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-embedding-service" + restart: unless-stopped + retriever: + image: opea/retriever-redis:latest + container_name: retriever-redis-server + depends_on: + - redis-vector-db + ports: + - "7000:7000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${REDIS_URL} + INDEX_NAME: ${INDEX_NAME} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-retriever-service" + restart: unless-stopped + tei-reranking-service: + image: ghcr.io/huggingface/text-embeddings-inference:1.2 + container_name: tei-reranking-server + ports: + - "8808:80" + volumes: + - "./data:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + command: --model-id ${RERANK_MODEL_ID} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + reranking: + image: opea/reranking-tei:latest + container_name: reranking-tei-server + depends_on: + - tei-reranking-service + ports: + - "8000:8000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-reranking-service" + restart: unless-stopped + tgi-service: + image: ghcr.io/huggingface/text-generation-inference:2.0 + container_name: tgi-server + ports: + - "8008:80" + volumes: + - "./data:/data" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + ipc: host + command: --model-id ${LLM_MODEL_ID} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + llm: + image: opea/llm-tgi:latest + container_name: llm-tgi-server + depends_on: + - tgi-service + ports: + - "9000:9000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-llm-service" + restart: unless-stopped + chaqna-backend-server: + image: opea/chatqna:latest + container_name: chatqna-backend-server + depends_on: + - redis-vector-db + - tei-embedding-service + - embedding + - retriever + - tei-reranking-service + - reranking + - tgi-service + - llm + ports: + - "8888:8888" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - EMBEDDING_SERVICE_HOST_IP=${EMBEDDING_SERVICE_HOST_IP} + - RETRIEVER_SERVICE_HOST_IP=${RETRIEVER_SERVICE_HOST_IP} + - RERANK_SERVICE_HOST_IP=${RERANK_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + ipc: host + restart: always + chaqna-ui-server: + image: opea/chatqna-ui:latest + container_name: chatqna-ui-server + depends_on: + - chaqna-backend-server + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + - UPLOAD_FILE_BASE_URL=${DATAPREP_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/ChatQnA/docker/xeon/docker_compose.yaml b/ChatQnA/docker/xeon/docker_compose.yaml index 4e93df4650..225735fbcb 100644 --- a/ChatQnA/docker/xeon/docker_compose.yaml +++ b/ChatQnA/docker/xeon/docker_compose.yaml @@ -85,6 +85,9 @@ services: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 command: --model-id ${RERANK_MODEL_ID} reranking: image: opea/reranking-tei:latest @@ -100,6 +103,8 @@ services: https_proxy: ${https_proxy} TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT} HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} LANGCHAIN_PROJECT: "opea-reranking-service" @@ -117,6 +122,8 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 command: --model-id ${LLM_MODEL_ID} llm: image: opea/llm-tgi:latest @@ -132,6 +139,8 @@ services: https_proxy: ${https_proxy} TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} LANGCHAIN_PROJECT: "opea-llm-service" diff --git a/ChatQnA/tests/test_chatqna_on_gaudi.sh b/ChatQnA/tests/test_chatqna_on_gaudi.sh index d7d86f486b..3467f36ae3 100644 --- a/ChatQnA/tests/test_chatqna_on_gaudi.sh +++ b/ChatQnA/tests/test_chatqna_on_gaudi.sh @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 set -e +echo "IMAGE_REPO=${IMAGE_REPO}" WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" @@ -58,9 +59,15 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env - # Start Docker Containers - # TODO: Replace the container name with a test-specific name + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers docker compose -f docker_compose.yaml up -d n=0 until [[ "$n" -ge 200 ]]; do @@ -212,14 +219,12 @@ function stop_docker() { function main() { stop_docker - begin_time=$(date +%s) - build_docker_images + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi start_time=$(date +%s) start_services end_time=$(date +%s) - minimal_duration=$((end_time-start_time)) - maximal_duration=$((end_time-begin_time)) - echo "Mega service start minimal duration is "$minimal_duration"s, maximal duration(including docker image build) is "$maximal_duration"s" + duration=$((end_time-start_time)) + echo "Mega service start duration is $duration s" validate_microservices validate_megaservice diff --git a/ChatQnA/tests/test_chatqna_on_xeon.sh b/ChatQnA/tests/test_chatqna_on_xeon.sh index 00afa06917..b9eba8449c 100644 --- a/ChatQnA/tests/test_chatqna_on_xeon.sh +++ b/ChatQnA/tests/test_chatqna_on_xeon.sh @@ -3,11 +3,32 @@ # SPDX-License-Identifier: Apache-2.0 set -e +echo "IMAGE_REPO=${IMAGE_REPO}" WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + + docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile . + docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile . + docker build -t opea/reranking-tei:latest -f comps/reranks/langchain/docker/Dockerfile . + docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile . + docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/chatqna:latest -f Dockerfile . + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile . + + docker images +} + function start_services() { cd $WORKPATH/docker/xeon @@ -30,11 +51,14 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env - # Replace the container name with a test-specific name - echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" - sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers docker compose -f docker_compose.yaml up -d n=0 @@ -188,12 +212,12 @@ function stop_docker() { function main() { stop_docker - - begin_time=$(date +%s) + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi + start_time=$(date +%s) start_services end_time=$(date +%s) - maximal_duration=$((end_time-begin_time)) - echo "Mega service start duration is "$maximal_duration"s" && sleep 1s + duration=$((end_time-start_time)) + echo "Mega service start duration is $duration s" && sleep 1s validate_microservices validate_megaservice diff --git a/ChatQnA/tests/test_manifest_on_gaudi.sh b/ChatQnA/tests/test_manifest_on_gaudi.sh new file mode 100755 index 0000000000..bb5bf52ecd --- /dev/null +++ b/ChatQnA/tests/test_manifest_on_gaudi.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +USER_ID=$(whoami) +LOG_PATH=/home/$(whoami)/logs +MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub +IMAGE_REPO=${IMAGE_REPO:-} +IMAGE_TAG=${IMAGE_TAG:-latest} + +function init_chatqna() { + # replace the mount dir "path: /mnt" with "path: $CHART_MOUNT" + find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/models#path: $MOUNT_DIR#g" {} \; + # replace megaservice image tag + find . -name '*.yaml' -type f -exec sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" {} \; + # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/" + find . -name '*.yaml' -type f -exec sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" {} \; + # set huggingface token + find . -name '*.yaml' -type f -exec sed -i "s#\${HUGGINGFACEHUB_API_TOKEN}#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \; +} + +function install_chatqna { + # replace namespace "default" with real namespace + find . -name '*.yaml' -type f -exec sed -i "s#default.svc#$NAMESPACE.svc#g" {} \; + # for very yaml file in yaml_files, apply it to the k8s cluster + yaml_files=("qna_configmap_gaudi" "redis-vector-db" "tei_embedding_gaudi_service" "tei_reranking_service" "tgi_gaudi_service" "retriever" "embedding" "reranking" "llm") + for yaml_file in ${yaml_files[@]}; do + kubectl apply -f $yaml_file.yaml -n $NAMESPACE + done + sleep 60 + kubectl apply -f chaqna-xeon-backend-server.yaml -n $NAMESPACE +} + +function validate_chatqna() { + max_retry=20 + # make sure microservice retriever is ready + # try to curl retriever-svc for max_retry times + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + for ((i=1; i<=max_retry; i++)) + do + curl http://retriever-svc.$NAMESPACE:7000/v1/retrieval -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \ + -H 'Content-Type: application/json' && break + sleep 10 + done + # make sure microservice tgi-svc is ready + for ((i=1; i<=max_retry; i++)) + do + curl http://tgi-gaudi-svc.$NAMESPACE:9009/generate -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ + -H 'Content-Type: application/json' && break + sleep 10 + done + # if i is bigger than max_retry, then exit with error + if [ $i -gt $max_retry ]; then + echo "Microservice failed, exit with error." + exit 1 + fi + + # check megaservice works + # generate a random logfile name to avoid conflict among multiple runners + LOGFILE=$LOG_PATH/curlmega_$NAMESPACE.log + curl http://chaqna-xeon-backend-server-svc.$NAMESPACE:8888/v1/chatqna -H "Content-Type: application/json" -d '{ + "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Megaservice failed, please check the logs in $LOGFILE!" + exit 1 + fi + + echo "Checking response results, make sure the output is reasonable. " + local status=false + if [[ -f $LOGFILE ]] && + [[ $(grep -c "billion" $LOGFILE) != 0 ]]; then + status=true + fi + if [ $status == false ]; then + echo "Response check failed, please check the logs in artifacts!" + exit 1 + else + echo "Response check succeed!" + fi +} + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +case "$1" in + init_ChatQnA) + pushd ChatQnA/kubernetes/manifests + init_chatqna + popd + ;; + install_ChatQnA) + pushd ChatQnA/kubernetes/manifests + NAMESPACE=$2 + install_chatqna + popd + ;; + validate_ChatQnA) + NAMESPACE=$2 + SERVICE_NAME=chaqna-xeon-backend-server-svc + validate_chatqna + ;; + *) + echo "Unknown function: $1" + ;; +esac diff --git a/CodeGen/tests/test_codegen_on_gaudi.sh b/CodeGen/tests/test_codegen_on_gaudi.sh index 824a30f5c0..ad06ca8896 100644 --- a/CodeGen/tests/test_codegen_on_gaudi.sh +++ b/CodeGen/tests/test_codegen_on_gaudi.sh @@ -38,8 +38,15 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers - # TODO: Replace the container name with a test-specific name docker compose -f docker_compose.yaml up -d sleep 2m # Waits 2 minutes @@ -141,7 +148,7 @@ function main() { stop_docker - build_docker_images + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi start_services validate_microservices diff --git a/CodeGen/tests/test_codegen_on_xeon.sh b/CodeGen/tests/test_codegen_on_xeon.sh index 6d4c1c9055..62f36eb92d 100644 --- a/CodeGen/tests/test_codegen_on_xeon.sh +++ b/CodeGen/tests/test_codegen_on_xeon.sh @@ -8,6 +8,22 @@ WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + + docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/codegen:latest -f Dockerfile . + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/codegen-ui:latest -f docker/Dockerfile . + + docker images +} + function start_services() { cd $WORKPATH/docker/xeon @@ -20,11 +36,14 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env - # Replace the container name with a test-specific name - echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" - sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers docker compose -f docker_compose.yaml up -d @@ -128,6 +147,7 @@ function main() { stop_docker + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi start_services validate_microservices diff --git a/CodeGen/tests/test_manifest_on_gaudi.sh b/CodeGen/tests/test_manifest_on_gaudi.sh new file mode 100755 index 0000000000..bf9a3b90c0 --- /dev/null +++ b/CodeGen/tests/test_manifest_on_gaudi.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +USER_ID=$(whoami) +LOG_PATH=/home/$(whoami)/logs +MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub +IMAGE_REPO=${IMAGE_REPO:-} +IMAGE_TAG=${IMAGE_TAG:-latest} + +function init_codegen() { + # executed under path manifest/codegen/xeon + # replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT" + find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt#path: $MOUNT_DIR#g" {} \; + # replace megaservice image tag + find . -name '*.yaml' -type f -exec sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" {} \; + # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/" + find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \; + # set huggingface token + find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \; +} + +function install_codegen { + echo "namespace is $NAMESPACE" + kubectl apply -f . -n $NAMESPACE +} + +function validate_codegen() { + ip_address=$(kubectl get svc $SERVICE_NAME -n $NAMESPACE -o jsonpath='{.spec.clusterIP}') + port=$(kubectl get svc $SERVICE_NAME -n $NAMESPACE -o jsonpath='{.spec.ports[0].port}') + echo "try to curl http://${ip_address}:${port}/v1/codegen..." + + # generate a random logfile name to avoid conflict among multiple runners + LOGFILE=$LOG_PATH/curlmega_$NAMESPACE.log + # Curl the Mega Service + curl http://${ip_address}:${port}/v1/codegen -H "Content-Type: application/json" \ + -d '{"messages": "def print_hello_world():"}' > $LOGFILE + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Megaservice codegen failed, please check the logs in $LOGFILE!" + exit 1 + fi + + echo "Checking response results, make sure the output is reasonable. " + local status=false + if [[ -f $LOGFILE ]] && \ + [[ $(grep -c "print" $LOGFILE) != 0 ]]; then + status=true + fi + + if [ $status == false ]; then + echo "Response check failed, please check the logs in artifacts!" + else + echo "Response check succeed!" + fi +} + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +case "$1" in + init_CodeGen) + pushd CodeGen/kubernetes/manifests/gaudi + init_codegen + popd + ;; + install_CodeGen) + pushd CodeGen/kubernetes/manifests/gaudi + NAMESPACE=$2 + install_codegen + popd + ;; + validate_CodeGen) + NAMESPACE=$2 + SERVICE_NAME=codegen + validate_codegen + ;; + *) + echo "Unknown function: $1" + ;; +esac diff --git a/CodeTrans/tests/test_codetrans_on_gaudi.sh b/CodeTrans/tests/test_codetrans_on_gaudi.sh index 5957325754..7b69e8649c 100644 --- a/CodeTrans/tests/test_codetrans_on_gaudi.sh +++ b/CodeTrans/tests/test_codetrans_on_gaudi.sh @@ -40,8 +40,15 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers - # TODO: Replace the container name with a test-specific name docker compose -f docker_compose.yaml up -d sleep 2m # Waits 2 minutes @@ -141,7 +148,7 @@ function main() { stop_docker - build_docker_images + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi start_services validate_microservices diff --git a/CodeTrans/tests/test_codetrans_on_xeon.sh b/CodeTrans/tests/test_codetrans_on_xeon.sh index e272f27e8f..6d652217fc 100644 --- a/CodeTrans/tests/test_codetrans_on_xeon.sh +++ b/CodeTrans/tests/test_codetrans_on_xeon.sh @@ -8,6 +8,22 @@ WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + + docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/codetrans:latest -f Dockerfile . + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/codetrans-ui:latest -f docker/Dockerfile . + + docker images +} + function start_services() { cd $WORKPATH/docker/xeon export http_proxy=${http_proxy} @@ -21,11 +37,14 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env - # Replace the container name with a test-specific name - echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" - sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers docker compose -f docker_compose.yaml up -d @@ -128,6 +147,7 @@ function main() { stop_docker + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi start_services validate_microservices diff --git a/DocSum/tests/test_docsum_on_gaudi.sh b/DocSum/tests/test_docsum_on_gaudi.sh index 8977006e7c..4c499bab35 100644 --- a/DocSum/tests/test_docsum_on_gaudi.sh +++ b/DocSum/tests/test_docsum_on_gaudi.sh @@ -38,8 +38,15 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/docsum:latest#image: opea/docsum:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/docsum-ui:latest#image: opea/docsum-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers - # TODO: Replace the container name with a test-specific name docker compose -f docker_compose.yaml up -d sleep 2m # Waits 2 minutes @@ -141,7 +148,7 @@ function main() { stop_docker - build_docker_images + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi start_services validate_microservices diff --git a/DocSum/tests/test_docsum_on_xeon.sh b/DocSum/tests/test_docsum_on_xeon.sh index 7bcf01c54c..f3cb192685 100644 --- a/DocSum/tests/test_docsum_on_xeon.sh +++ b/DocSum/tests/test_docsum_on_xeon.sh @@ -8,6 +8,22 @@ WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + + docker build --no-cache -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/summarization/tgi/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . + + docker images +} + function start_services() { cd $WORKPATH/docker/xeon @@ -20,11 +36,14 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env - # Replace the container name with a test-specific name - echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" - sed -i "s#image: opea/docsum:latest#image: opea/docsum:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/docsum-ui:latest#image: opea/docsum-ui:${IMAGE_TAG}#g" docker_compose.yaml - sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/docsum:latest#image: opea/docsum:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/docsum-ui:latest#image: opea/docsum-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + # Start Docker Containers docker compose -f docker_compose.yaml up -d @@ -127,6 +146,7 @@ function main() { stop_docker + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi start_services validate_microservices diff --git a/README.md b/README.md index c2cb68c969..0a2629f499 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ All the examples are well-validated on Intel platforms. In addition, these examp BGE-Base Redis TGI TEI - Xeon/Gaudi2 + Xeon/Gaudi2/GPU Chatbot diff --git a/Translation/docker/ui/svelte/.env b/Translation/docker/ui/svelte/.env index d92ea96f4b..4df6832379 100644 --- a/Translation/docker/ui/svelte/.env +++ b/Translation/docker/ui/svelte/.env @@ -1 +1 @@ -BASE_URL = 'http://backend_address:8888/v1/translation' \ No newline at end of file +BASE_URL = 'http://10.7.5.135:8888/v1/translation' \ No newline at end of file diff --git a/Translation/docker/ui/svelte/src/lib/shared/Network.ts b/Translation/docker/ui/svelte/src/lib/shared/Network.ts index d1029c014a..489550818a 100644 --- a/Translation/docker/ui/svelte/src/lib/shared/Network.ts +++ b/Translation/docker/ui/svelte/src/lib/shared/Network.ts @@ -12,32 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. import { env } from "$env/dynamic/public"; +import { SSE } from "sse.js"; const BASE_URL = env.BASE_URL; -async function fetchPostRes(url, init) { - try { - const response = await fetch(url, init); - if (!response.ok) throw response.status; - return await response.json(); - } catch (error) { - console.error("network error: ", error); - return undefined; - } -} - export async function fetchLanguageResponse(input: string, transform: string, transTo: string) { - const url = `${BASE_URL}/v1/translation`; - const transData = { + let payload = {}; + let url = ""; + + payload = { language_from: transform, language_to: transTo, source_language: input, }; + url = `${BASE_URL}`; - const init: RequestInit = { - method: "POST", - body: JSON.stringify(transData), - }; - - return fetchPostRes(url, init); + return new SSE(url, { + headers: { "Content-Type": "application/json" }, + payload: JSON.stringify(payload), + }); } diff --git a/Translation/docker/ui/svelte/src/routes/+page.svelte b/Translation/docker/ui/svelte/src/routes/+page.svelte index ec76c9c4a2..92a092398a 100644 --- a/Translation/docker/ui/svelte/src/routes/+page.svelte +++ b/Translation/docker/ui/svelte/src/routes/+page.svelte @@ -32,15 +32,48 @@ let output: string = ""; let loading = false; - async function handelTranslate() { - loading = true; - const res = await fetchLanguageResponse(input, langFrom, langTo); - if (res) { - output = res.target_language; - loading = false; - } + function decodeEscapedBytes(str: string): string { + const byteArray = str + .split("\\x") + .slice(1) + .map((byte) => parseInt(byte, 16)); + return new TextDecoder("utf-8").decode(new Uint8Array(byteArray)); + } + + function decodeUnicode(str: string): string { + return str.replace(/\\u[\dA-Fa-f]{4}/g, (match) => { + return String.fromCharCode(parseInt(match.replace(/\\u/g, ""), 16)); + }); } + const handelTranslate = async () => { + loading = true; + output = ""; + const eventSource = await fetchLanguageResponse(input, langFrom, langTo); + + eventSource.addEventListener("message", (e: any) => { + let Msg = e.data; + console.log("Msg", Msg); + + if (Msg.startsWith("b")) { + let trimmedData = Msg.slice(2, -1); + + if (/\\x[\dA-Fa-f]{2}/.test(trimmedData)) { + trimmedData = decodeEscapedBytes(trimmedData); + } else if (/\\u[\dA-Fa-f]{4}/.test(trimmedData)) { + trimmedData = decodeUnicode(trimmedData); + } + + if (trimmedData !== "") { + output += trimmedData.replace(/\\n/g, "\n"); + } + } else if (Msg === "[DONE]") { + loading = false; + } + }); + eventSource.stream(); + }; + let timer; $: if ((input || langFrom || langTo) && input !== "") {