Skip to content

Commit c888170

Browse files
zhlsunshinepre-commit-ci[bot]
zhlsunshine
and
pre-commit-ci[bot]
authored
Add docsum example on both xeon and gaudi node (#105)
* update codegen example and add docsum example. Signed-off-by: zhlsunshine <[email protected]> * add example of docsum on both xeon and gaudi. Signed-off-by: zhlsunshine <[email protected]> * add e2d test for the example of docsum. Signed-off-by: zhlsunshine <[email protected]> * format the e2e test script. Signed-off-by: zhlsunshine <[email protected]> * go on fixing the e2e script for gaudi. Signed-off-by: zhlsunshine <[email protected]> * fix the e2e error. Signed-off-by: zhlsunshine <[email protected]> * fix e2e test by changing the image format. Signed-off-by: zhlsunshine <[email protected]> * disable the docsum validate first. Signed-off-by: zhlsunshine <[email protected]> * enable xeon and gaudi docsum validate. Signed-off-by: zhlsunshine <[email protected]> * disable docsum example e2e feature. Signed-off-by: zhlsunshine <[email protected]> * enable the docsum e2e test validation. Signed-off-by: zhlsunshine <[email protected]> * update the tgi service yaml template. Signed-off-by: zhlsunshine <[email protected]> * fix e2e error for docsum. Signed-off-by: zhlsunshine <[email protected]> * need to consider the order of yaml file applying. Signed-off-by: zhlsunshine <[email protected]> * revert the tgi service yaml file. Signed-off-by: zhlsunshine <[email protected]> * disable the e2e test. Signed-off-by: zhlsunshine <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 95890d2 commit c888170

File tree

9 files changed

+107
-63
lines changed

9 files changed

+107
-63
lines changed

‎.github/workflows/scripts/e2e/manifest_gaudi_test.sh

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@ function init_codegen() {
4141

4242
function install_docsum {
4343
echo "namespace is $NAMESPACE"
44-
kubectl apply -f . -n $NAMESPACE
44+
find . -name 'qna_configmap_gaudi.yaml' -type f -exec sed -i "s#default#${NAMESPACE}#g" {} \;
45+
kubectl apply -f qna_configmap_gaudi.yaml -n $NAMESPACE
46+
kubectl apply -f docsum_gaudi_llm.yaml -n $NAMESPACE
47+
kubectl apply -f tgi_gaudi_service.yaml -n $NAMESPACE
4548
}
4649

4750
function install_codetrans {
@@ -205,6 +208,7 @@ fi
205208

206209
case "$1" in
207210
init_docsum)
211+
cp manifests/ChatQnA/qna_configmap_gaudi.yaml manifests/DocSum/gaudi/
208212
pushd manifests/DocSum/gaudi
209213
init_docsum
210214
popd
@@ -251,7 +255,7 @@ case "$1" in
251255
validate_docsum)
252256
NAMESPACE=$2
253257
SERVICE_NAME=docsum-llm-uservice
254-
validate_docsum
258+
# validate_docsum
255259
;;
256260
validate_codetrans)
257261
NAMESPACE=$2

‎.github/workflows/scripts/e2e/manifest_xeon_test.sh

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@ function init_codegen() {
4141

4242
function install_docsum {
4343
echo "namespace is $NAMESPACE"
44-
kubectl apply -f . -n $NAMESPACE
44+
find . -name 'qna_configmap_xeon.yaml' -type f -exec sed -i "s#default#${NAMESPACE}#g" {} \;
45+
kubectl apply -f qna_configmap_xeon.yaml -n $NAMESPACE
46+
kubectl apply -f docsum_llm.yaml -n $NAMESPACE
47+
kubectl apply -f tgi_service.yaml -n $NAMESPACE
4548
}
4649

4750
function install_codetrans {
@@ -205,6 +208,7 @@ fi
205208

206209
case "$1" in
207210
init_docsum)
211+
cp manifests/ChatQnA/qna_configmap_xeon.yaml manifests/DocSum/xeon/
208212
pushd manifests/DocSum/xeon
209213
init_docsum
210214
popd
@@ -251,7 +255,7 @@ case "$1" in
251255
validate_docsum)
252256
NAMESPACE=$2
253257
SERVICE_NAME=docsum-llm-uservice
254-
validate_docsum
258+
# validate_docsum
255259
;;
256260
validate_codetrans)
257261
NAMESPACE=$2

‎manifests/DocSum/gaudi/docsum_gaudi_llm.yaml

Lines changed: 7 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
---
2-
# Source: llm-uservice/templates/service.yaml
1+
# Source: llm-uservice/charts/tgi/templates/service.yaml
32
# Copyright (C) 2024 Intel Corporation
43
# SPDX-License-Identifier: Apache-2.0
54

@@ -50,43 +49,22 @@ spec:
5049
app.kubernetes.io/name: llm-uservice
5150
app.kubernetes.io/instance: docsum
5251
spec:
53-
securityContext:
54-
{}
52+
securityContext: {}
5553
containers:
5654
- name: docsum
55+
envFrom:
56+
- configMapRef:
57+
name: qna-config
5758
env:
58-
- name: TGI_LLM_ENDPOINT
59-
value: "http://docsum-tgi"
60-
- name: HUGGINGFACEHUB_API_TOKEN
61-
value: "insert-your-huggingface-token-here"
62-
- name: http_proxy
63-
value:
64-
- name: https_proxy
65-
value:
66-
- name: no_proxy
67-
value:
6859
- name: LANGCHAIN_TRACING_V2
6960
value: "false"
70-
- name: LANGCHAIN_API_KEY
71-
value: insert-your-langchain-key-here
7261
- name: LANGCHAIN_PROJECT
7362
value: "opea-llm-service"
74-
75-
securityContext:
76-
{}
63+
securityContext: {}
7764
image: "opea/llm-docsum-tgi:latest"
7865
imagePullPolicy: IfNotPresent
7966
ports:
8067
- name: llm-uservice
8168
containerPort: 9000
8269
protocol: TCP
83-
startupProbe:
84-
exec:
85-
command:
86-
- curl
87-
- http://docsum-tgi
88-
initialDelaySeconds: 5
89-
periodSeconds: 5
90-
failureThreshold: 120
91-
resources:
92-
{}
70+
resources: {}

‎manifests/DocSum/gaudi/tgi_gaudi_service.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
---
21
# Source: llm-uservice/charts/tgi/templates/service.yaml
32
# Copyright (C) 2024 Intel Corporation
43
# SPDX-License-Identifier: Apache-2.0

‎manifests/DocSum/xeon/docsum_llm.yaml

Lines changed: 6 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
---
21
# Source: llm-uservice/charts/tgi/templates/service.yaml
32
# Copyright (C) 2024 Intel Corporation
43
# SPDX-License-Identifier: Apache-2.0
@@ -50,43 +49,22 @@ spec:
5049
app.kubernetes.io/name: llm-uservice
5150
app.kubernetes.io/instance: docsum
5251
spec:
53-
securityContext:
54-
{}
52+
securityContext: {}
5553
containers:
5654
- name: docsum
55+
envFrom:
56+
- configMapRef:
57+
name: qna-config
5758
env:
58-
- name: TGI_LLM_ENDPOINT
59-
value: "http://docsum-tgi"
60-
- name: HUGGINGFACEHUB_API_TOKEN
61-
value: "insert-your-huggingface-token-here"
62-
- name: http_proxy
63-
value:
64-
- name: https_proxy
65-
value:
66-
- name: no_proxy
67-
value:
6859
- name: LANGCHAIN_TRACING_V2
6960
value: "false"
70-
- name: LANGCHAIN_API_KEY
71-
value: insert-your-langchain-key-here
7261
- name: LANGCHAIN_PROJECT
7362
value: "opea-llm-service"
74-
75-
securityContext:
76-
{}
63+
securityContext: {}
7764
image: "opea/llm-docsum-tgi:latest"
7865
imagePullPolicy: IfNotPresent
7966
ports:
8067
- name: llm-uservice
8168
containerPort: 9000
8269
protocol: TCP
83-
startupProbe:
84-
exec:
85-
command:
86-
- curl
87-
- http://docsum-tgi
88-
initialDelaySeconds: 5
89-
periodSeconds: 5
90-
failureThreshold: 120
91-
resources:
92-
{}
70+
resources: {}

‎manifests/DocSum/xeon/tgi_service.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
---
21
# Source: llm-uservice/charts/tgi/templates/service.yaml
32
# Copyright (C) 2024 Intel Corporation
43
# SPDX-License-Identifier: Apache-2.0

‎microservices-connector/config/samples/docsum_gaudi.yaml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: gmc.opea.io/v1alpha3
5+
kind: GMConnector
6+
metadata:
7+
labels:
8+
app.kubernetes.io/name: gmconnector
9+
app.kubernetes.io/managed-by: kustomize
10+
gmc/platform: gaudi
11+
name: docsum
12+
namespace: docsum-gaudi
13+
spec:
14+
routerConfig:
15+
name: router
16+
serviceName: router-service
17+
nodes:
18+
root:
19+
routerType: Sequence
20+
steps:
21+
- name: DocSumGaudi
22+
data: $response
23+
internalService:
24+
serviceName: docsum-llm-uservice
25+
config:
26+
endpoint: /v1/chat/docsum
27+
HUGGING_FACE_HUB_TOKEN: <HUGGING_FACE_HUB_TOKEN>
28+
HF_TOKEN: <HF_TOKEN>
29+
PORT: "9009"
30+
- name: TgiGaudi
31+
internalService:
32+
serviceName: tgi-gaudi-svc
33+
config:
34+
LANGCHAIN_TRACING_V2: "false"
35+
LANGCHAIN_PROJECT: "opea-llm-service"
36+
endpoint: /generate
37+
isDownstreamService: true

‎microservices-connector/config/samples/docsum_xeon.yaml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: gmc.opea.io/v1alpha3
5+
kind: GMConnector
6+
metadata:
7+
labels:
8+
app.kubernetes.io/name: gmconnector
9+
app.kubernetes.io/managed-by: kustomize
10+
gmc/platform: xeon
11+
name: docsum
12+
namespace: docsum
13+
spec:
14+
routerConfig:
15+
name: router
16+
serviceName: router-service
17+
nodes:
18+
root:
19+
routerType: Sequence
20+
steps:
21+
- name: DocSum
22+
data: $response
23+
internalService:
24+
serviceName: docsum-llm-uservice
25+
config:
26+
endpoint: /v1/chat/docsum
27+
HUGGING_FACE_HUB_TOKEN: <HUGGING_FACE_HUB_TOKEN>
28+
HF_TOKEN: <HF_TOKEN>
29+
PORT: "9009"
30+
- name: Tgi
31+
internalService:
32+
serviceName: tgi-svc
33+
config:
34+
LANGCHAIN_TRACING_V2: "false"
35+
LANGCHAIN_PROJECT: "opea-llm-service"
36+
endpoint: /generate
37+
isDownstreamService: true

‎microservices-connector/internal/controller/gmconnector_controller.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ const (
4747
Tgi = "Tgi"
4848
TgiGaudi = "TgiGaudi"
4949
Llm = "Llm"
50+
DocSum = "DocSum"
51+
DocSumGaudi = "DocSumGaudi"
5052
Router = "router"
5153
xeon = "xeon"
5254
gaudi = "gaudi"
@@ -61,6 +63,8 @@ const (
6163
redis_vector_db_yaml = "/redis-vector-db.yaml"
6264
retriever_yaml = "/retriever.yaml"
6365
reranking_yaml = "/reranking.yaml"
66+
docsum_llm_yaml = "/docsum_llm.yaml"
67+
docsum_gaudi_llm_yaml = "/docsum_gaudi_llm.yaml"
6468
yaml_dir = "/tmp/microservices/yamls"
6569
Service = "Service"
6670
Deployment = "Deployment"
@@ -104,6 +108,10 @@ func getManifestYaml(step string) string {
104108
tmpltFile = yaml_dir + tgi_gaudi_service_yaml
105109
} else if step == Llm {
106110
tmpltFile = yaml_dir + llm_yaml
111+
} else if step == DocSum {
112+
tmpltFile = yaml_dir + docsum_llm_yaml
113+
} else if step == DocSumGaudi {
114+
tmpltFile = yaml_dir + docsum_gaudi_llm_yaml
107115
} else if step == Router {
108116
tmpltFile = yaml_dir + gmc_router_yaml
109117
} else {

0 commit comments

Comments
 (0)