diff --git a/.github/workflows/Translation.yml b/.github/workflows/Translation.yml deleted file mode 100644 index c4b867dbc8..0000000000 --- a/.github/workflows/Translation.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -name: Translation-test - -on: - pull_request: - branches: [main] - types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - paths: - - Translation/** - - "!**.md" - - "!**/ui/**" - - .github/workflows/Translation.yml - workflow_dispatch: - -# If there is a new commit, the previous jobs will be canceled -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - Translation: - runs-on: aise-cluster - strategy: - matrix: - job_name: ["langchain"] - fail-fast: false - steps: - - name: Clean Up Working Directory - run: sudo rm -rf ${{github.workspace}}/* - - - name: Checkout out Repo - uses: actions/checkout@v4 - with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Run Test - env: - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - run: | - cd ${{ github.workspace }}/Translation/tests - bash test_${{ matrix.job_name }}_inference.sh - - - name: Publish pipeline artifact - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.job_name }} - path: ${{ github.workspace }}/Translation/tests/*.log diff --git a/.github/workflows/scripts/build_push.sh b/.github/workflows/scripts/build_push.sh index 4dd0ecb43b..89246006fd 100755 --- a/.github/workflows/scripts/build_push.sh +++ b/.github/workflows/scripts/build_push.sh @@ -46,14 +46,14 @@ function docker_build() { # $1 is like "apple orange pear" for MEGA_SVC in $1; do case $MEGA_SVC in - "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum") + "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"Translation") cd $MEGA_SVC/docker IMAGE_NAME="$(getImagenameFromMega $MEGA_SVC)" docker_build ${IMAGE_NAME} cd ui docker_build ${IMAGE_NAME}-ui docker/Dockerfile ;; - "AudioQnA"|"SearchQnA"|"Translation"|"VisualQnA") + "AudioQnA"|"SearchQnA"|"VisualQnA") echo "Not supported yet" ;; *) diff --git a/Translation/README.md b/Translation/README.md index 3b06ad192a..546596f902 100644 --- a/Translation/README.md +++ b/Translation/README.md @@ -1,51 +1,21 @@ -# Language Translation +# Translation Application Language Translation is the communication of the meaning of a source-language text by means of an equivalent target-language text. -The workflow falls into the following architecture: +Translation architecture shows below: ![architecture](./assets/img/translation_architecture.png) -# Start Backend Service +This Translation use case performs Language Translation Inference on Intel Gaudi2 or Intel XEON Scalable Processors. The Intel Gaudi2 accelerator supports both training and inference for deep learning models in particular for LLMs. Please visit [Habana AI products](https://habana.ai/products) for more details. -1. Start the TGI Service to deploy your LLM +# Deploy Translation Service -```sh -cd serving/tgi_gaudi -bash build_docker.sh -bash launch_tgi_service.sh -``` +The Translation service can be effortlessly deployed on either Intel Gaudi2 or Intel XEON Scalable Processors. -`launch_tgi_service.sh` the script uses `8080` as the TGI service's port by default. Please replace it if any port conflicts detected. +## Deploy Translation on Gaudi -2. Start the Language Translation Service +Refer to the [Gaudi Guide](./docker/gaudi/README.md) for instructions on deploying Translation on Gaudi. -```sh -cd langchain/docker -bash build_docker.sh -docker run -it --name translation_server --net=host --ipc=host -e TGI_ENDPOINT=${TGI_ENDPOINT} -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -e SERVER_PORT=8000 -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} translation:latest bash -``` +## Deploy Translation on Xeon -**Note**: Set the following parameters before running the above command - -- `TGI_ENDPOINT`: The endpoint of your TGI service, usually equal to `:`. -- `HUGGINGFACEHUB_API_TOKEN`: Your HuggingFace hub API token, usually generated [here](https://huggingface.co/settings/tokens). -- `SERVER_PORT`: The port of the Translation service on the host. - -3. Quick Test - -```sh -curl http://localhost:8000/v1/translation \ - -X POST \ - -d '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' \ - -H 'Content-Type: application/json' -``` - -The shortcodes of languages are also supported: - -```sh -curl http://localhost:8000/v1/translation \ - -X POST \ - -d '{"language_from": "de","language_to": "en","source_language": "Maschinelles Lernen"}' \ - -H 'Content-Type: application/json' -``` +Refer to the [Xeon Guide](./docker/xeon/README.md) for instructions on deploying Translation on Xeon. diff --git a/Translation/deprecated/README.md b/Translation/deprecated/README.md new file mode 100644 index 0000000000..366866231f --- /dev/null +++ b/Translation/deprecated/README.md @@ -0,0 +1,51 @@ +# Language Translation + +Language Translation is the communication of the meaning of a source-language text by means of an equivalent target-language text. + +The workflow falls into the following architecture: + +![architecture](../assets/img/translation_architecture.png) + +# Start Backend Service + +1. Start the TGI Service to deploy your LLM + +```sh +cd serving/tgi_gaudi +bash build_docker.sh +bash launch_tgi_service.sh +``` + +`launch_tgi_service.sh` the script uses `8080` as the TGI service's port by default. Please replace it if any port conflicts detected. + +2. Start the Language Translation Service + +```sh +cd langchain/docker +bash build_docker.sh +docker run -it --name translation_server --net=host --ipc=host -e TGI_ENDPOINT=${TGI_ENDPOINT} -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -e SERVER_PORT=8000 -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} translation:latest bash +``` + +**Note**: Set the following parameters before running the above command + +- `TGI_ENDPOINT`: The endpoint of your TGI service, usually equal to `:`. +- `HUGGINGFACEHUB_API_TOKEN`: Your HuggingFace hub API token, usually generated [here](https://huggingface.co/settings/tokens). +- `SERVER_PORT`: The port of the Translation service on the host. + +3. Quick Test + +```sh +curl http://localhost:8000/v1/translation \ + -X POST \ + -d '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' \ + -H 'Content-Type: application/json' +``` + +The shortcodes of languages are also supported: + +```sh +curl http://localhost:8000/v1/translation \ + -X POST \ + -d '{"language_from": "de","language_to": "en","source_language": "Maschinelles Lernen"}' \ + -H 'Content-Type: application/json' +``` diff --git a/Translation/langchain/docker/Dockerfile b/Translation/deprecated/langchain/docker/Dockerfile similarity index 100% rename from Translation/langchain/docker/Dockerfile rename to Translation/deprecated/langchain/docker/Dockerfile diff --git a/Translation/langchain/docker/build_docker.sh b/Translation/deprecated/langchain/docker/build_docker.sh similarity index 100% rename from Translation/langchain/docker/build_docker.sh rename to Translation/deprecated/langchain/docker/build_docker.sh diff --git a/Translation/langchain/docker/requirements.txt b/Translation/deprecated/langchain/docker/requirements.txt similarity index 100% rename from Translation/langchain/docker/requirements.txt rename to Translation/deprecated/langchain/docker/requirements.txt diff --git a/Translation/langchain/docker/translation-app/prompts.py b/Translation/deprecated/langchain/docker/translation-app/prompts.py similarity index 100% rename from Translation/langchain/docker/translation-app/prompts.py rename to Translation/deprecated/langchain/docker/translation-app/prompts.py diff --git a/Translation/langchain/docker/translation-app/server.py b/Translation/deprecated/langchain/docker/translation-app/server.py similarity index 100% rename from Translation/langchain/docker/translation-app/server.py rename to Translation/deprecated/langchain/docker/translation-app/server.py diff --git a/Translation/serving/tgi_gaudi/Dockerfile b/Translation/deprecated/serving/tgi_gaudi/Dockerfile similarity index 100% rename from Translation/serving/tgi_gaudi/Dockerfile rename to Translation/deprecated/serving/tgi_gaudi/Dockerfile diff --git a/Translation/serving/tgi_gaudi/build_docker.sh b/Translation/deprecated/serving/tgi_gaudi/build_docker.sh similarity index 100% rename from Translation/serving/tgi_gaudi/build_docker.sh rename to Translation/deprecated/serving/tgi_gaudi/build_docker.sh diff --git a/Translation/serving/tgi_gaudi/launch_tgi_service.sh b/Translation/deprecated/serving/tgi_gaudi/launch_tgi_service.sh similarity index 100% rename from Translation/serving/tgi_gaudi/launch_tgi_service.sh rename to Translation/deprecated/serving/tgi_gaudi/launch_tgi_service.sh diff --git a/Translation/tests/test_langchain_inference.sh b/Translation/deprecated/tests/test_langchain_inference.sh similarity index 100% rename from Translation/tests/test_langchain_inference.sh rename to Translation/deprecated/tests/test_langchain_inference.sh diff --git a/Translation/docker/Dockerfile b/Translation/docker/Dockerfile new file mode 100644 index 0000000000..1c4bde0a70 --- /dev/null +++ b/Translation/docker/Dockerfile @@ -0,0 +1,42 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FROM langchain/langchain:latest + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim \ + git + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +RUN cd /home/user/ && \ + git clone https://github.com/opea-project/GenAIComps.git + +RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \ + pip install -r /home/user/GenAIComps/requirements.txt + +COPY ./translation.py /home/user/translation.py + +ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps + +USER user + +WORKDIR /home/user + +ENTRYPOINT ["python", "translation.py"] diff --git a/Translation/docker/gaudi/README.md b/Translation/docker/gaudi/README.md new file mode 100644 index 0000000000..b16fcaa8a1 --- /dev/null +++ b/Translation/docker/gaudi/README.md @@ -0,0 +1,104 @@ +# Build MegaService of Translation on Gaudi + +This document outlines the deployment process for a Translation application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as We will publish the Docker images to Docker Hub, it will simplify the deployment process for this service. + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally. This step can be ignored after the Docker images published to Docker hub. + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +``` + +### 1. Build LLM Image + +```bash +docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . +``` + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `translation.py` Python script. Build the MegaService Docker image using the command below: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/Translation/docker +docker build -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Construct the frontend Docker image using the command below: + +```bash +cd GenAIExamples/Translation/docker/ui/ +docker build -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +1. `ghcr.io/huggingface/tgi-gaudi:1.2.1` +2. `opea/gen-ai-comps:llm-tgi-gaudi-server` +3. `opea/gen-ai-comps:translation-megaservice-server` +4. `opea/gen-ai-comps:translation-ui-server` + +## 🚀 Start Microservices + +### Setup Environment Variables + +Since the `docker_compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export LLM_MODEL_ID="haoranxu/ALMA-13B" +export TGI_LLM_ENDPOINT="http://${host_ip}:8008" +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export MEGA_SERVICE_HOST_IP=${host_ip} +export LLM_SERVICE_HOST_IP=${host_ip} +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/translation" +``` + +Note: Please replace with `host_ip` with you external IP address, do not use localhost. + +### Start Microservice Docker Containers + +```bash +docker compose -f docker_compose.yaml up -d +``` + +### Validate Microservices + +1. TGI Service + +```bash +curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' +``` + +2. LLM Microservice + +```bash +curl http://${host_ip}:9000/v1/chat/completions \ + -X POST \ + -d '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' \ + -H 'Content-Type: application/json' +``` + +3. MegaService + +```bash +curl http://${host_ip}:8888/v1/translation -H "Content-Type: application/json" -d '{ + "language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' +``` + +Following the validation of all aforementioned microservices, we are now prepared to construct a mega-service. + +## 🚀 Launch the UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. +![project-screenshot](../../assets/img/trans_ui_init.png) +![project-screenshot](../../assets/img/trans_ui_select.png) diff --git a/Translation/docker/gaudi/docker_compose.yaml b/Translation/docker/gaudi/docker_compose.yaml new file mode 100644 index 0000000000..87a044b24e --- /dev/null +++ b/Translation/docker/gaudi/docker_compose.yaml @@ -0,0 +1,76 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +services: + tgi_service: + image: ghcr.io/huggingface/tgi-gaudi:1.2.1 + container_name: tgi_gaudi_service + ports: + - "8008:80" + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + volumes: + - "./data:/data" + command: --model-id ${LLM_MODEL_ID} + llm: + image: opea/llm-tgi:latest + container_name: llm-tgi-gaudi-server + depends_on: + - tgi_service + ports: + - "9000:9000" + ipc: host + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + restart: unless-stopped + translation-gaudi-backend-server: + image: opea/translation:latest + container_name: translation-gaudi-backend-server + depends_on: + - tgi_service + - llm + ports: + - "8888:8888" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + ipc: host + restart: always + translation-gaudi-ui-server: + image: opea/translation-ui:latest + container_name: translation-gaudi-ui-server + depends_on: + - translation-gaudi-backend-server + ports: + - "5173:5173" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/Translation/docker/translation.py b/Translation/docker/translation.py new file mode 100644 index 0000000000..338661bfe5 --- /dev/null +++ b/Translation/docker/translation.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os + +from comps import MicroService, ServiceOrchestrator, ServiceType, TranslationGateway + +MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0") +MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 8888) +LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0") +LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000) + + +class TranslationService: + def __init__(self, host="0.0.0.0", port=8000): + self.host = host + self.port = port + self.megaservice = ServiceOrchestrator() + + def add_remote_service(self): + llm = MicroService( + name="llm", + host=LLM_SERVICE_HOST_IP, + port=LLM_SERVICE_PORT, + endpoint="/v1/chat/completions", + use_remote_service=True, + service_type=ServiceType.LLM, + ) + self.megaservice.add(llm) + self.gateway = TranslationGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port) + + +if __name__ == "__main__": + translation = TranslationService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT) + translation.add_remote_service() diff --git a/Translation/ui/docker/Dockerfile b/Translation/docker/ui/docker/Dockerfile similarity index 98% rename from Translation/ui/docker/Dockerfile rename to Translation/docker/ui/docker/Dockerfile index ac2bb7da31..1d5115f4b5 100644 --- a/Translation/ui/docker/Dockerfile +++ b/Translation/docker/ui/docker/Dockerfile @@ -23,4 +23,4 @@ RUN npm run build EXPOSE 5173 # Run the front-end application in preview mode -CMD ["npm", "run", "preview", "--", "--port", "5173", "--host", "0.0.0.0"] \ No newline at end of file +CMD ["npm", "run", "preview", "--", "--port", "5173", "--host", "0.0.0.0"] diff --git a/Translation/docker/ui/svelte/.env b/Translation/docker/ui/svelte/.env new file mode 100644 index 0000000000..4df6832379 --- /dev/null +++ b/Translation/docker/ui/svelte/.env @@ -0,0 +1 @@ +BASE_URL = 'http://10.7.5.135:8888/v1/translation' \ No newline at end of file diff --git a/Translation/ui/svelte/.gitignore b/Translation/docker/ui/svelte/.gitignore similarity index 100% rename from Translation/ui/svelte/.gitignore rename to Translation/docker/ui/svelte/.gitignore diff --git a/Translation/ui/svelte/.npmrc b/Translation/docker/ui/svelte/.npmrc similarity index 100% rename from Translation/ui/svelte/.npmrc rename to Translation/docker/ui/svelte/.npmrc diff --git a/Translation/ui/svelte/README.md b/Translation/docker/ui/svelte/README.md similarity index 82% rename from Translation/ui/svelte/README.md rename to Translation/docker/ui/svelte/README.md index 7ce136f512..3db5bf7628 100644 --- a/Translation/ui/svelte/README.md +++ b/Translation/docker/ui/svelte/README.md @@ -2,8 +2,8 @@ ### 📸 Project Screenshots -![project-screenshot](../../assets/img/trans_ui_init.png) -![project-screenshot](../../assets/img/trans_ui_select.png) +![project-screenshot](../../../assets/img/trans_ui_init.png) +![project-screenshot](../../../assets/img/trans_ui_select.png)

🧐 Features

diff --git a/Translation/ui/svelte/package.json b/Translation/docker/ui/svelte/package.json similarity index 78% rename from Translation/ui/svelte/package.json rename to Translation/docker/ui/svelte/package.json index 027dad1954..41dbb477b8 100644 --- a/Translation/ui/svelte/package.json +++ b/Translation/docker/ui/svelte/package.json @@ -3,32 +3,23 @@ "version": "0.0.1", "scripts": { "dev": "vite dev", - "build": "vite build && npm run package", + "build": "vite build", "preview": "vite preview", "package": "svelte-kit sync && svelte-package && publint", "prepublishOnly": "npm run package", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch" }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "svelte": "./dist/index.js" - } - }, - "files": [ - "dist", - "!dist/**/*.test.*", - "!dist/**/*.spec.*" - ], "peerDependencies": { "svelte": "^4.0.0" }, "devDependencies": { + "@playwright/test": "^1.44.1", "@sveltejs/adapter-auto": "^3.0.0", "@sveltejs/kit": "^2.0.0", "@sveltejs/package": "^2.0.0", "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@types/prismjs": "^1.26.3", "autoprefixer": "^10.4.16", "flowbite": "^2.3.0", "flowbite-svelte": "^0.38.5", @@ -38,15 +29,15 @@ "publint": "^0.1.9", "svelte": "^4.2.7", "svelte-check": "^3.6.0", + "svelte-highlight": "^7.6.0", "tailwindcss": "^3.3.6", "tslib": "^2.4.1", "typescript": "^5.0.0", "vite": "^5.0.11" }, - "svelte": "./dist/index.js", - "types": "./dist/index.d.ts", "type": "module", "dependencies": { + "prismjs": "^1.29.0", "sse.js": "^0.6.1", "svelte-notifications": "^0.9.98" } diff --git a/Translation/ui/svelte/postcss.config.cjs b/Translation/docker/ui/svelte/postcss.config.cjs similarity index 100% rename from Translation/ui/svelte/postcss.config.cjs rename to Translation/docker/ui/svelte/postcss.config.cjs diff --git a/Translation/ui/svelte/src/app.d.ts b/Translation/docker/ui/svelte/src/app.d.ts similarity index 100% rename from Translation/ui/svelte/src/app.d.ts rename to Translation/docker/ui/svelte/src/app.d.ts diff --git a/Translation/ui/svelte/src/app.html b/Translation/docker/ui/svelte/src/app.html similarity index 100% rename from Translation/ui/svelte/src/app.html rename to Translation/docker/ui/svelte/src/app.html diff --git a/Translation/ui/svelte/src/app.pcss b/Translation/docker/ui/svelte/src/app.pcss similarity index 100% rename from Translation/ui/svelte/src/app.pcss rename to Translation/docker/ui/svelte/src/app.pcss diff --git a/Translation/ui/svelte/src/lib/assets/loadingAnimation.svelte b/Translation/docker/ui/svelte/src/lib/assets/loadingAnimation.svelte similarity index 100% rename from Translation/ui/svelte/src/lib/assets/loadingAnimation.svelte rename to Translation/docker/ui/svelte/src/lib/assets/loadingAnimation.svelte diff --git a/Translation/ui/svelte/src/lib/assets/summaryLogo.svelte b/Translation/docker/ui/svelte/src/lib/assets/summaryLogo.svelte similarity index 100% rename from Translation/ui/svelte/src/lib/assets/summaryLogo.svelte rename to Translation/docker/ui/svelte/src/lib/assets/summaryLogo.svelte diff --git a/Translation/ui/svelte/src/lib/header.svelte b/Translation/docker/ui/svelte/src/lib/header.svelte similarity index 100% rename from Translation/ui/svelte/src/lib/header.svelte rename to Translation/docker/ui/svelte/src/lib/header.svelte diff --git a/Translation/ui/svelte/src/lib/shared/Network.ts b/Translation/docker/ui/svelte/src/lib/shared/Network.ts similarity index 62% rename from Translation/ui/svelte/src/lib/shared/Network.ts rename to Translation/docker/ui/svelte/src/lib/shared/Network.ts index f77add7959..489550818a 100644 --- a/Translation/ui/svelte/src/lib/shared/Network.ts +++ b/Translation/docker/ui/svelte/src/lib/shared/Network.ts @@ -12,32 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. import { env } from "$env/dynamic/public"; +import { SSE } from "sse.js"; -const BASIC_URL = env.BASIC_URL; - -async function fetchPostRes(url, init) { - try { - const response = await fetch(url, init); - if (!response.ok) throw response.status; - return await response.json(); - } catch (error) { - console.error("network error: ", error); - return undefined; - } -} +const BASE_URL = env.BASE_URL; export async function fetchLanguageResponse(input: string, transform: string, transTo: string) { - const url = `${BASIC_URL}/v1/translation`; - const transData = { + let payload = {}; + let url = ""; + + payload = { language_from: transform, language_to: transTo, source_language: input, }; + url = `${BASE_URL}`; - const init: RequestInit = { - method: "POST", - body: JSON.stringify(transData), - }; - - return fetchPostRes(url, init); + return new SSE(url, { + headers: { "Content-Type": "application/json" }, + payload: JSON.stringify(payload), + }); } diff --git a/Translation/ui/svelte/src/lib/shared/constant.ts b/Translation/docker/ui/svelte/src/lib/shared/constant.ts similarity index 100% rename from Translation/ui/svelte/src/lib/shared/constant.ts rename to Translation/docker/ui/svelte/src/lib/shared/constant.ts diff --git a/Translation/ui/svelte/src/routes/+layout.svelte b/Translation/docker/ui/svelte/src/routes/+layout.svelte similarity index 100% rename from Translation/ui/svelte/src/routes/+layout.svelte rename to Translation/docker/ui/svelte/src/routes/+layout.svelte diff --git a/Translation/ui/svelte/src/routes/+page.svelte b/Translation/docker/ui/svelte/src/routes/+page.svelte similarity index 74% rename from Translation/ui/svelte/src/routes/+page.svelte rename to Translation/docker/ui/svelte/src/routes/+page.svelte index ec76c9c4a2..92a092398a 100644 --- a/Translation/ui/svelte/src/routes/+page.svelte +++ b/Translation/docker/ui/svelte/src/routes/+page.svelte @@ -32,15 +32,48 @@ let output: string = ""; let loading = false; - async function handelTranslate() { - loading = true; - const res = await fetchLanguageResponse(input, langFrom, langTo); - if (res) { - output = res.target_language; - loading = false; - } + function decodeEscapedBytes(str: string): string { + const byteArray = str + .split("\\x") + .slice(1) + .map((byte) => parseInt(byte, 16)); + return new TextDecoder("utf-8").decode(new Uint8Array(byteArray)); + } + + function decodeUnicode(str: string): string { + return str.replace(/\\u[\dA-Fa-f]{4}/g, (match) => { + return String.fromCharCode(parseInt(match.replace(/\\u/g, ""), 16)); + }); } + const handelTranslate = async () => { + loading = true; + output = ""; + const eventSource = await fetchLanguageResponse(input, langFrom, langTo); + + eventSource.addEventListener("message", (e: any) => { + let Msg = e.data; + console.log("Msg", Msg); + + if (Msg.startsWith("b")) { + let trimmedData = Msg.slice(2, -1); + + if (/\\x[\dA-Fa-f]{2}/.test(trimmedData)) { + trimmedData = decodeEscapedBytes(trimmedData); + } else if (/\\u[\dA-Fa-f]{4}/.test(trimmedData)) { + trimmedData = decodeUnicode(trimmedData); + } + + if (trimmedData !== "") { + output += trimmedData.replace(/\\n/g, "\n"); + } + } else if (Msg === "[DONE]") { + loading = false; + } + }); + eventSource.stream(); + }; + let timer; $: if ((input || langFrom || langTo) && input !== "") { diff --git a/Translation/ui/svelte/src/routes/types.d.ts b/Translation/docker/ui/svelte/src/routes/types.d.ts similarity index 100% rename from Translation/ui/svelte/src/routes/types.d.ts rename to Translation/docker/ui/svelte/src/routes/types.d.ts diff --git a/Translation/ui/svelte/static/favicon.png b/Translation/docker/ui/svelte/static/favicon.png similarity index 100% rename from Translation/ui/svelte/static/favicon.png rename to Translation/docker/ui/svelte/static/favicon.png diff --git a/Translation/ui/svelte/svelte.config.js b/Translation/docker/ui/svelte/svelte.config.js similarity index 100% rename from Translation/ui/svelte/svelte.config.js rename to Translation/docker/ui/svelte/svelte.config.js diff --git a/Translation/ui/svelte/tailwind.config.cjs b/Translation/docker/ui/svelte/tailwind.config.cjs similarity index 100% rename from Translation/ui/svelte/tailwind.config.cjs rename to Translation/docker/ui/svelte/tailwind.config.cjs diff --git a/Translation/ui/svelte/tsconfig.json b/Translation/docker/ui/svelte/tsconfig.json similarity index 100% rename from Translation/ui/svelte/tsconfig.json rename to Translation/docker/ui/svelte/tsconfig.json diff --git a/Translation/ui/svelte/vite.config.ts b/Translation/docker/ui/svelte/vite.config.ts similarity index 100% rename from Translation/ui/svelte/vite.config.ts rename to Translation/docker/ui/svelte/vite.config.ts diff --git a/Translation/docker/xeon/README.md b/Translation/docker/xeon/README.md new file mode 100644 index 0000000000..ee01e90797 --- /dev/null +++ b/Translation/docker/xeon/README.md @@ -0,0 +1,111 @@ +# Build Mega Service of Translation on Xeon + +This document outlines the deployment process for a Translation application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `llm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. + +## 🚀 Apply Xeon Server on AWS + +To apply a Xeon server on AWS, start by creating an AWS account if you don't have one already. Then, head to the [EC2 Console](https://console.aws.amazon.com/ec2/v2/home) to begin the process. Within the EC2 service, select the Amazon EC2 M7i or M7i-flex instance type to leverage the power of 4th Generation Intel Xeon Scalable processors. These instances are optimized for high-performance computing and demanding workloads. + +For detailed information about these instance types, you can refer to this [link](https://aws.amazon.com/ec2/instance-types/m7i/). Once you've chosen the appropriate instance type, proceed with configuring your instance settings, including network configurations, security groups, and storage options. + +After launching your instance, you can connect to it using SSH (for Linux instances) or Remote Desktop Protocol (RDP) (for Windows instances). From there, you'll have full access to your Xeon server, allowing you to install, configure, and manage your applications as needed. + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +``` + +### 1. Build LLM Image + +```bash +docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . +``` + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `translation.py` Python script. Build MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/Translation/docker +docker build -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build frontend Docker image via below command: + +```bash +cd GenAIExamples/Translation/docker/ui +docker build -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/gen-ai-comps:llm-tgi-server` +2. `opea/gen-ai-comps:translation-megaservice-server` +3. `opea/gen-ai-comps:translation-ui-server` + +## 🚀 Start Microservices + +### Setup Environment Variables + +Since the `docker_compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export LLM_MODEL_ID="haoranxu/ALMA-13B" +export TGI_LLM_ENDPOINT="http://${host_ip}:8008" +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export MEGA_SERVICE_HOST_IP=${host_ip} +export LLM_SERVICE_HOST_IP=${host_ip} +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/translation" +``` + +Note: Please replace with `host_ip` with you external IP address, do not use localhost. + +### Start Microservice Docker Containers + +```bash +docker compose -f docker_compose.yaml up -d +``` + +### Validate Microservices + +1. TGI Service + +```bash +curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ + -H 'Content-Type: application/json' +``` + +2. LLM Microservice + +```bash +curl http://${host_ip}:9000/v1/chat/completions \ + -X POST \ + -d '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' \ + -H 'Content-Type: application/json' +``` + +3. MegaService + +```bash +curl http://${host_ip}:8888/v1/translation -H "Content-Type: application/json" -d '{ + "language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' +``` + +Following the validation of all aforementioned microservices, we are now prepared to construct a mega-service. + +## 🚀 Launch the UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. +![project-screenshot](../../assets/img/trans_ui_init.png) +![project-screenshot](../../assets/img/trans_ui_select.png) diff --git a/Translation/docker/xeon/docker_compose.yaml b/Translation/docker/xeon/docker_compose.yaml new file mode 100644 index 0000000000..4f627223e7 --- /dev/null +++ b/Translation/docker/xeon/docker_compose.yaml @@ -0,0 +1,79 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +services: + tgi_service: + image: ghcr.io/huggingface/text-generation-inference:1.4 + container_name: tgi_service + ports: + - "8008:80" + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + volumes: + - "./data:/data" + shm_size: 1g + command: --model-id ${LLM_MODEL_ID} + llm: + image: opea/llm-tgi:latest + container_name: llm-tgi-server + depends_on: + - tgi_service + ports: + - "9000:9000" + ipc: host + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-llm-service" + restart: unless-stopped + translation-xeon-backend-server: + image: opea/translation:latest + container_name: translation-xeon-backend-server + depends_on: + - tgi_service + - llm + ports: + - "8888:8888" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + ipc: host + restart: always + translation-xeon-ui-server: + image: opea/translation-ui:latest + container_name: translation-xeon-ui-server + depends_on: + - translation-xeon-backend-server + ports: + - "5173:5173" + environment: + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always +networks: + default: + driver: bridge diff --git a/Translation/kubernetes/README.md b/Translation/kubernetes/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/kubernetes/helm-charts/README.md b/Translation/kubernetes/helm-charts/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/kubernetes/manifests/README.md b/Translation/kubernetes/manifests/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/kubernetes/service-mesh/README.md b/Translation/kubernetes/service-mesh/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Translation/tests/test_translation_on_gaudi.sh b/Translation/tests/test_translation_on_gaudi.sh new file mode 100644 index 0000000000..de33a4f637 --- /dev/null +++ b/Translation/tests/test_translation_on_gaudi.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . + + cd $WORKPATH/docker + docker build --no-cache -t opea/translation:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + + docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1 + + cd $WORKPATH/docker/ui + docker build --no-cache -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . + + docker images +} + +function start_services() { + cd $WORKPATH/docker/gaudi + + export LLM_MODEL_ID="haoranxu/ALMA-13B" + export TGI_LLM_ENDPOINT="http://${ip_address}:8008" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/translation" + + # Start Docker Containers + # TODO: Replace the container name with a test-specific name + docker compose -f docker_compose.yaml up -d + + sleep 2m # Waits 2 minutes +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + + + +function validate_microservices() { + # Check if the microservices are running correctly. + # TODO: Any results check required?? + sleep 3m + # tgi gaudi service + validate_services \ + "${ip_address}:8008/generate" \ + "generated_text" \ + "tgi-gaudi" \ + "tgi_gaudi_service" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "llm-tgi-gaudi-server" \ + '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/translation" \ + "translation" \ + "mega-translation" \ + "translation-gaudi-backend-server" \ + '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' +} + +function validate_frontend() { + cd $WORKPATH/docker/ui/svelte + local conda_env_name="Translation_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + conda remove -n ${conda_env_name} --all -y + conda create -n ${conda_env_name} python=3.12 -y + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs -y && npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi + +} + +function stop_docker() { + cd $WORKPATH/docker/gaudi + container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + done +} + +function main() { + + stop_docker + + build_docker_images + start_services + + validate_microservices + validate_megaservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/Translation/tests/test_translation_on_xeon.sh b/Translation/tests/test_translation_on_xeon.sh new file mode 100644 index 0000000000..809bd1c069 --- /dev/null +++ b/Translation/tests/test_translation_on_xeon.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function start_services() { + cd $WORKPATH/docker/xeon + + export LLM_MODEL_ID="haoranxu/ALMA-13B" + export TGI_LLM_ENDPOINT="http://${ip_address}:8008" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/translation" + + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/translation:latest#image: opea/translation:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/translation-ui:latest#image: opea/translation-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + + # Start Docker Containers + # TODO: Replace the container name with a test-specific name + docker compose -f docker_compose.yaml up -d + + sleep 2m # Waits 2 minutes +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + # TODO: Any results check required?? + sleep 3m + # tgi for llm service + validate_services \ + "${ip_address}:8008/generate" \ + "generated_text" \ + "tgi" \ + "tgi_service" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "llm-tgi-server" \ + '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/translation" \ + "translation" \ + "mega-translation" \ + "translation-xeon-backend-server" \ + '{"language_from": "Chinese","language_to": "English","source_language": "我爱机器翻译。"}' +} + +function validate_frontend() { + cd $WORKPATH/docker/ui/svelte + local conda_env_name="Translation_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + conda remove -n ${conda_env_name} --all -y + conda create -n ${conda_env_name} python=3.12 -y + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs -y && npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi + +} + +function stop_docker() { + cd $WORKPATH/docker/xeon + container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + done +} + +function main() { + + stop_docker + + start_services + + validate_microservices + validate_megaservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/Translation/translation.yaml b/Translation/translation.yaml new file mode 100644 index 0000000000..b27fe7b7bb --- /dev/null +++ b/Translation/translation.yaml @@ -0,0 +1,44 @@ + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +opea_micro_services: + tgi_service: + host: ${TGI_SERVICE_IP} + ports: ${TGI_SERVICE_PORT} + image: ghcr.io/huggingface/tgi-gaudi:1.2.1 + volumes: + - "./data:/data" + runtime: habana + cap_add: + - SYS_NICE + ipc: host + environment: + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + model-id: ${LLM_MODEL_ID} + llm: + host: ${LLM_SERVICE_HOST_IP} + ports: ${LLM_SERVICE_PORT} + image: opea/llm-tgi:latest + endpoint: /v1/chat/completions + environment: + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + ui: + host: ${UI_SERVICE_HOST_IP} + ports: + - "5173:5173" + image: opea/translation-ui:latest + environment: + - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + +opea_mega_service: + host: ${MEGA_SERVICE_HOST_IP} + ports: ${MEGA_SERVICE_PORT} + image: opea/translation:latest + endpoint: /v1/translation + mega_flow: + - llm diff --git a/Translation/ui/svelte/.env b/Translation/ui/svelte/.env deleted file mode 100644 index 4fe9cb96ad..0000000000 --- a/Translation/ui/svelte/.env +++ /dev/null @@ -1 +0,0 @@ -BASIC_URL = 'http://x.x.x.x:yyyy' \ No newline at end of file