Skip to content

Commit d0b028d

Browse files
zhlsunshinezhlsunshine
authored
Add new example of Translation for GenAIExample (#455)
* Add Translation GMC pipeline Signed-off-by: zhlsunshine <[email protected]>
1 parent 8b60948 commit d0b028d

File tree

6 files changed

+364
-0
lines changed

6 files changed

+364
-0
lines changed

‎SearchQnA/kubernetes/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ In the below example we illustrate on Xeon.
1414
export APP_NAMESPACE=CT
1515
kubectl create ns $APP_NAMESPACE
1616
sed -i "s|namespace: searchqa|namespace: $APP_NAMESPACE|g" ./searchQnA_xeon.yaml
17+
sed -i "s|insert-your-google-api-key-here|$GOOGLE_API_KEY|g" ./searchQnA_xeon.yaml
18+
sed -i "s|insert-your-google-cse-id-here|$GOOGLE_CSE_ID|g" ./searchQnA_xeon.yaml
1719
kubectl apply -f ./searchQnA_xeon.yaml
1820
```
1921

‎Translation/kubernetes/README.md

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<h1 align="center" id="title">Deploy Translation in a Kubernetes Cluster</h1>
2+
3+
This document outlines the deployment process for a Code Generation (Translation) application that utilizes the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice components on Intel Xeon servers and Gaudi machines.
4+
5+
Please install GMC in your Kubernetes cluster, if you have not already done so, by following the steps in Section "Getting Started" at [GMC Install](https://github.com/opea-project/GenAIInfra/tree/main/microservices-connector#readme). We will soon publish images to Docker Hub, at which point no builds will be required, further simplifying install.
6+
7+
If you have only Intel Xeon machines you could use the translation_xeon.yaml file or if you have a Gaudi cluster you could use translation_gaudi.yaml
8+
In the below example we illustrate on Xeon.
9+
10+
## Deploy the RAG application
11+
12+
1. Create the desired namespace if it does not already exist and deploy the application
13+
```bash
14+
export APP_NAMESPACE=CT
15+
kubectl create ns $APP_NAMESPACE
16+
sed -i "s|namespace: translation|namespace: $APP_NAMESPACE|g" ./translation_xeon.yaml
17+
kubectl apply -f ./translation_xeon.yaml
18+
```
19+
20+
2. Check if the application is up and ready
21+
```bash
22+
kubectl get pods -n $APP_NAMESPACE
23+
```
24+
25+
3. Deploy a client pod for testing
26+
```bash
27+
kubectl create deployment client-test -n $APP_NAMESPACE --image=python:3.8.13 -- sleep infinity
28+
```
29+
30+
4. Check that client pod is ready
31+
```bash
32+
kubectl get pods -n $APP_NAMESPACE
33+
```
34+
35+
5. Send request to application
36+
```bash
37+
export CLIENT_POD=$(kubectl get pod -n $APP_NAMESPACE -l app=client-test -o jsonpath={.items..metadata.name})
38+
export accessUrl=$(kubectl get gmc -n $APP_NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='translation')].status.accessUrl}")
39+
kubectl exec "$CLIENT_POD" -n $APP_NAMESPACE -- curl $accessUrl -X POST -d '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' -H 'Content-Type: application/json' > $LOG_PATH/gmc_translation.log
40+
```

‎Translation/kubernetes/translation_gaudi.yaml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: gmc.opea.io/v1alpha3
5+
kind: GMConnector
6+
metadata:
7+
labels:
8+
app.kubernetes.io/name: gmconnector
9+
app.kubernetes.io/managed-by: kustomize
10+
gmc/platform: gaudi
11+
name: translation
12+
namespace: translation
13+
spec:
14+
routerConfig:
15+
name: router
16+
serviceName: router-service
17+
nodes:
18+
root:
19+
routerType: Sequence
20+
steps:
21+
- name: Llm
22+
data: $response
23+
internalService:
24+
serviceName: llm-service
25+
config:
26+
endpoint: /v1/chat/completions
27+
TGI_LLM_ENDPOINT: tgi-gaudi-svc
28+
- name: TgiGaudi
29+
internalService:
30+
serviceName: tgi-gaudi-svc
31+
config:
32+
MODEL_ID: haoranxu/ALMA-13B
33+
endpoint: /generate
34+
isDownstreamService: true

‎Translation/kubernetes/translation_xeon.yaml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: gmc.opea.io/v1alpha3
5+
kind: GMConnector
6+
metadata:
7+
labels:
8+
app.kubernetes.io/name: gmconnector
9+
app.kubernetes.io/managed-by: kustomize
10+
gmc/platform: xeon
11+
name: translation
12+
namespace: translation
13+
spec:
14+
routerConfig:
15+
name: router
16+
serviceName: router-service
17+
nodes:
18+
root:
19+
routerType: Sequence
20+
steps:
21+
- name: Llm
22+
data: $response
23+
internalService:
24+
serviceName: llm-service
25+
config:
26+
endpoint: /v1/chat/completions
27+
TGI_LLM_ENDPOINT: tgi-service
28+
- name: Tgi
29+
internalService:
30+
serviceName: tgi-service
31+
config:
32+
MODEL_ID: haoranxu/ALMA-13B
33+
endpoint: /generate
34+
isDownstreamService: true

‎Translation/tests/test_gmc_on_gaudi.sh

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -xe
6+
USER_ID=$(whoami)
7+
LOG_PATH=/home/$(whoami)/logs
8+
MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
9+
IMAGE_REPO=${IMAGE_REPO:-}
10+
11+
function install_translation() {
12+
kubectl create ns $APP_NAMESPACE
13+
sed -i "s|namespace: translation|namespace: $APP_NAMESPACE|g" ./translation_gaudi.yaml
14+
kubectl apply -f ./translation_gaudi.yaml
15+
16+
# Wait until the router service is ready
17+
echo "Waiting for the translation router service to be ready..."
18+
wait_until_pod_ready "translation router" $APP_NAMESPACE "router-service"
19+
output=$(kubectl get pods -n $APP_NAMESPACE)
20+
echo $output
21+
}
22+
23+
function validate_translation() {
24+
# deploy client pod for testing
25+
kubectl create deployment client-test -n $APP_NAMESPACE --image=python:3.8.13 -- sleep infinity
26+
27+
# wait for client pod ready
28+
wait_until_pod_ready "client-test" $APP_NAMESPACE "client-test"
29+
# giving time to populating data
30+
sleep 60
31+
32+
kubectl get pods -n $APP_NAMESPACE
33+
# send request to translation
34+
export CLIENT_POD=$(kubectl get pod -n $APP_NAMESPACE -l app=client-test -o jsonpath={.items..metadata.name})
35+
echo "$CLIENT_POD"
36+
accessUrl=$(kubectl get gmc -n $APP_NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='translation')].status.accessUrl}")
37+
kubectl exec "$CLIENT_POD" -n $APP_NAMESPACE -- curl $accessUrl -X POST -d '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' -H 'Content-Type: application/json' > $LOG_PATH/gmc_translation.log
38+
exit_code=$?
39+
if [ $exit_code -ne 0 ]; then
40+
echo "chatqna failed, please check the logs in ${LOG_PATH}!"
41+
exit 1
42+
fi
43+
44+
echo "Checking response results, make sure the output is reasonable. "
45+
local status=false
46+
if [[ -f $LOG_PATH/gmc_translation.log ]] && \
47+
[[ $(grep -c "[DONE]" $LOG_PATH/gmc_translation.log) != 0 ]]; then
48+
status=true
49+
fi
50+
if [ $status == false ]; then
51+
if [[ -f $LOG_PATH/gmc_translation.log ]]; then
52+
cat $LOG_PATH/gmc_translation.log
53+
fi
54+
echo "Response check failed, please check the logs in artifacts!"
55+
cat $LOG_PATH/gmc_translation.log
56+
exit 1
57+
else
58+
echo "Response check succeed!"
59+
fi
60+
}
61+
62+
function wait_until_pod_ready() {
63+
echo "Waiting for the $1 to be ready..."
64+
max_retries=30
65+
retry_count=0
66+
while ! is_pod_ready $2 $3; do
67+
if [ $retry_count -ge $max_retries ]; then
68+
echo "$1 is not ready after waiting for a significant amount of time"
69+
get_gmc_controller_logs
70+
exit 1
71+
fi
72+
echo "$1 is not ready yet. Retrying in 10 seconds..."
73+
sleep 10
74+
output=$(kubectl get pods -n $2)
75+
echo $output
76+
retry_count=$((retry_count + 1))
77+
done
78+
}
79+
80+
function is_pod_ready() {
81+
if [ "$2" == "gmc-controller" ]; then
82+
pod_status=$(kubectl get pods -n $1 -o jsonpath='{.items[].status.conditions[?(@.type=="Ready")].status}')
83+
else
84+
pod_status=$(kubectl get pods -n $1 -l app=$2 -o jsonpath='{.items[].status.conditions[?(@.type=="Ready")].status}')
85+
fi
86+
if [ "$pod_status" == "True" ]; then
87+
return 0
88+
else
89+
return 1
90+
fi
91+
}
92+
93+
function get_gmc_controller_logs() {
94+
# Fetch the name of the pod with the app-name gmc-controller in the specified namespace
95+
pod_name=$(kubectl get pods -n $SYSTEM_NAMESPACE -l control-plane=gmc-controller -o jsonpath='{.items[0].metadata.name}')
96+
97+
# Check if the pod name was found
98+
if [ -z "$pod_name" ]; then
99+
echo "No pod found with app-name gmc-controller in namespace $SYSTEM_NAMESPACE"
100+
return 1
101+
fi
102+
103+
# Get the logs of the found pod
104+
echo "Fetching logs for pod $pod_name in namespace $SYSTEM_NAMESPACE..."
105+
kubectl logs $pod_name -n $SYSTEM_NAMESPACE
106+
}
107+
108+
if [ $# -eq 0 ]; then
109+
echo "Usage: $0 <function_name>"
110+
exit 1
111+
fi
112+
113+
case "$1" in
114+
install_Translation)
115+
pushd Translation/kubernetes
116+
install_translation
117+
popd
118+
;;
119+
validate_Translation)
120+
pushd Translation/kubernetes
121+
validate_translation
122+
popd
123+
;;
124+
*)
125+
echo "Unknown function: $1"
126+
;;
127+
esac

‎Translation/tests/test_gmc_on_xeon.sh

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -xe
6+
USER_ID=$(whoami)
7+
LOG_PATH=/home/$(whoami)/logs
8+
MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
9+
IMAGE_REPO=${IMAGE_REPO:-}
10+
11+
function install_translation() {
12+
kubectl create ns $APP_NAMESPACE
13+
sed -i "s|namespace: translation|namespace: $APP_NAMESPACE|g" ./translation_xeon.yaml
14+
kubectl apply -f ./translation_xeon.yaml
15+
16+
# Wait until the router service is ready
17+
echo "Waiting for the translation router service to be ready..."
18+
wait_until_pod_ready "translation router" $APP_NAMESPACE "router-service"
19+
output=$(kubectl get pods -n $APP_NAMESPACE)
20+
echo $output
21+
}
22+
23+
function validate_translation() {
24+
# deploy client pod for testing
25+
kubectl create deployment client-test -n $APP_NAMESPACE --image=python:3.8.13 -- sleep infinity
26+
27+
# wait for client pod ready
28+
wait_until_pod_ready "client-test" $APP_NAMESPACE "client-test"
29+
# giving time to populating data
30+
sleep 60
31+
32+
kubectl get pods -n $APP_NAMESPACE
33+
# send request to translation
34+
export CLIENT_POD=$(kubectl get pod -n $APP_NAMESPACE -l app=client-test -o jsonpath={.items..metadata.name})
35+
echo "$CLIENT_POD"
36+
accessUrl=$(kubectl get gmc -n $APP_NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='translation')].status.accessUrl}")
37+
kubectl exec "$CLIENT_POD" -n $APP_NAMESPACE -- curl $accessUrl -X POST -d '{"query":"Translate this from Chinese to English:\nChinese: 我爱机器翻译。\nEnglish:"}' -H 'Content-Type: application/json' > $LOG_PATH/gmc_translation.log
38+
exit_code=$?
39+
if [ $exit_code -ne 0 ]; then
40+
echo "chatqna failed, please check the logs in ${LOG_PATH}!"
41+
exit 1
42+
fi
43+
44+
echo "Checking response results, make sure the output is reasonable. "
45+
local status=false
46+
if [[ -f $LOG_PATH/gmc_translation.log ]] && \
47+
[[ $(grep -c "[DONE]" $LOG_PATH/gmc_translation.log) != 0 ]]; then
48+
status=true
49+
fi
50+
if [ $status == false ]; then
51+
if [[ -f $LOG_PATH/gmc_translation.log ]]; then
52+
cat $LOG_PATH/gmc_translation.log
53+
fi
54+
echo "Response check failed, please check the logs in artifacts!"
55+
cat $LOG_PATH/gmc_translation.log
56+
exit 1
57+
else
58+
echo "Response check succeed!"
59+
fi
60+
}
61+
62+
function wait_until_pod_ready() {
63+
echo "Waiting for the $1 to be ready..."
64+
max_retries=30
65+
retry_count=0
66+
while ! is_pod_ready $2 $3; do
67+
if [ $retry_count -ge $max_retries ]; then
68+
echo "$1 is not ready after waiting for a significant amount of time"
69+
get_gmc_controller_logs
70+
exit 1
71+
fi
72+
echo "$1 is not ready yet. Retrying in 10 seconds..."
73+
sleep 10
74+
output=$(kubectl get pods -n $2)
75+
echo $output
76+
retry_count=$((retry_count + 1))
77+
done
78+
}
79+
80+
function is_pod_ready() {
81+
if [ "$2" == "gmc-controller" ]; then
82+
pod_status=$(kubectl get pods -n $1 -o jsonpath='{.items[].status.conditions[?(@.type=="Ready")].status}')
83+
else
84+
pod_status=$(kubectl get pods -n $1 -l app=$2 -o jsonpath='{.items[].status.conditions[?(@.type=="Ready")].status}')
85+
fi
86+
if [ "$pod_status" == "True" ]; then
87+
return 0
88+
else
89+
return 1
90+
fi
91+
}
92+
93+
function get_gmc_controller_logs() {
94+
# Fetch the name of the pod with the app-name gmc-controller in the specified namespace
95+
pod_name=$(kubectl get pods -n $SYSTEM_NAMESPACE -l control-plane=gmc-controller -o jsonpath='{.items[0].metadata.name}')
96+
97+
# Check if the pod name was found
98+
if [ -z "$pod_name" ]; then
99+
echo "No pod found with app-name gmc-controller in namespace $SYSTEM_NAMESPACE"
100+
return 1
101+
fi
102+
103+
# Get the logs of the found pod
104+
echo "Fetching logs for pod $pod_name in namespace $SYSTEM_NAMESPACE..."
105+
kubectl logs $pod_name -n $SYSTEM_NAMESPACE
106+
}
107+
108+
if [ $# -eq 0 ]; then
109+
echo "Usage: $0 <function_name>"
110+
exit 1
111+
fi
112+
113+
case "$1" in
114+
install_Translation)
115+
pushd Translation/kubernetes
116+
install_translation
117+
popd
118+
;;
119+
validate_Translation)
120+
pushd Translation/kubernetes
121+
validate_translation
122+
popd
123+
;;
124+
*)
125+
echo "Unknown function: $1"
126+
;;
127+
esac

0 commit comments

Comments
 (0)