Skip to content

Commit e32a514

Browse files
chensuyuechensuyue
authored
Add build docker image option for test scripts (#312)
Signed-off-by: chensuyue <[email protected]>
1 parent e948a7f commit e32a514

12 files changed

+174
-59
lines changed

‎.github/workflows/docker-compose-e2e.yml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,13 @@ jobs:
6161
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
6262
example: ${{ matrix.example }}
6363
hardware: ${{ matrix.hardware }}
64-
IMAGE_REPO: ${{needs.mega-image-build.outputs.image_repo}}
65-
IMAGE_TAG: ${{needs.mega-image-build.outputs.image_tag}}
64+
IMAGE_TAG: ${{ needs.mega-image-build.outputs.image_tag }}
65+
IMAGE_REPO_GAUDI: ${{ vars.IMAGE_REPO_GAUDI }}
66+
IMAGE_REPO_XEON: ${{ vars.IMAGE_REPO_XEON }}
6667
run: |
6768
cd ${{ github.workspace }}/$example/tests
69+
if [ "$hardware" == "gaudi" ]; then IMAGE_REPO=$IMAGE_REPO_GAUDI; else IMAGE_REPO=$IMAGE_REPO_XEON; fi
70+
export IMAGE_REPO=${IMAGE_REPO}
6871
example_l=$(echo $example | tr '[:upper:]' '[:lower:]')
6972
if [ -f test_${example_l}_on_${hardware}.sh ]; then timeout 30m bash test_${example_l}_on_${hardware}.sh; else echo "Test script not found, skip test!"; fi
7073

‎.github/workflows/manifest-e2e.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ jobs:
5757

5858
- name: Set variables
5959
run: |
60-
echo "IMAGE_REPO=${{needs.mega-image-build.outputs.image_repo}}" >> $GITHUB_ENV
60+
if [ ${{ matrix.hardware }} == "gaudi" ]; then IMAGE_REPO=${{ vars.IMAGE_REPO_GAUDI }}; else IMAGE_REPO=${{ vars.IMAGE_REPO_XEON }}; fi
61+
echo "IMAGE_REPO=${IMAGE_REPO}" >> $GITHUB_ENV
6162
echo "IMAGE_TAG=${{needs.mega-image-build.outputs.image_tag}}" >> $GITHUB_ENV
6263
lower_example=$(echo "${{ matrix.example }}" | tr '[:upper:]' '[:lower:]')
6364
echo "NAMESPACE=$lower_example-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV

‎.github/workflows/reuse-image-build.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,5 @@ jobs:
6060
if [ -z "${{ env.IMAGE_REPO }}" ]; then
6161
IMAGE_REPO=$OPEA_IMAGE_REPO
6262
fi
63-
echo "IMAGE_REPO=${IMAGE_REPO}"
6463
echo "IMAGE_TAG=${IMAGE_TAG}"
65-
echo "image_repo=$IMAGE_REPO" >> $GITHUB_OUTPUT
6664
echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT

‎.github/workflows/scripts/build_push.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ function docker_build() {
4646
# $1 is like "apple orange pear"
4747
for MEGA_SVC in $1; do
4848
case $MEGA_SVC in
49-
"ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"SearchQnA")
49+
"ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum")
5050
cd $MEGA_SVC/docker
5151
IMAGE_NAME="$(getImagenameFromMega $MEGA_SVC)"
5252
docker_build ${IMAGE_NAME}
@@ -55,7 +55,6 @@ for MEGA_SVC in $1; do
5555
;;
5656
"AudioQnA"|"SearchQnA"|"Translation"|"VisualQnA")
5757
echo "Not supported yet"
58-
exit 1
5958
;;
6059
*)
6160
echo "Unknown function: $MEGA_SVC"

‎ChatQnA/tests/test_chatqna_on_gaudi.sh

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -e
6+
echo "IMAGE_REPO=${IMAGE_REPO}"
67

78
WORKPATH=$(dirname "$PWD")
89
LOG_PATH="$WORKPATH/tests"
@@ -58,11 +59,14 @@ function start_services() {
5859

5960
sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
6061

61-
# Replace the container name with a test-specific name
62-
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
63-
sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
64-
sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
65-
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
62+
if [[ "$IMAGE_REPO" != "" ]]; then
63+
# Replace the container name with a test-specific name
64+
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
65+
sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
66+
sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
67+
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
68+
fi
69+
6670
# Start Docker Containers
6771
docker compose -f docker_compose.yaml up -d
6872
n=0
@@ -215,14 +219,12 @@ function stop_docker() {
215219
function main() {
216220

217221
stop_docker
218-
begin_time=$(date +%s)
219-
# build_docker_images
220-
# start_time=$(date +%s)
222+
if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
223+
start_time=$(date +%s)
221224
start_services
222225
end_time=$(date +%s)
223-
# minimal_duration=$((end_time-start_time))
224-
maximal_duration=$((end_time-begin_time))
225-
echo "Mega service start duration is "$maximal_duration"s"
226+
duration=$((end_time-start_time))
227+
echo "Mega service start duration is $duration s"
226228

227229
validate_microservices
228230
validate_megaservice

‎ChatQnA/tests/test_chatqna_on_xeon.sh

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,32 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -e
6+
echo "IMAGE_REPO=${IMAGE_REPO}"
67

78
WORKPATH=$(dirname "$PWD")
89
LOG_PATH="$WORKPATH/tests"
910
ip_address=$(hostname -I | awk '{print $1}')
1011

12+
function build_docker_images() {
13+
cd $WORKPATH
14+
git clone https://github.com/opea-project/GenAIComps.git
15+
cd GenAIComps
16+
17+
docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile .
18+
docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile .
19+
docker build -t opea/reranking-tei:latest -f comps/reranks/langchain/docker/Dockerfile .
20+
docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
21+
docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile .
22+
23+
cd $WORKPATH/docker
24+
docker build --no-cache -t opea/chatqna:latest -f Dockerfile .
25+
26+
cd $WORKPATH/docker/ui
27+
docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile .
28+
29+
docker images
30+
}
31+
1132
function start_services() {
1233
cd $WORKPATH/docker/xeon
1334

@@ -30,11 +51,14 @@ function start_services() {
3051

3152
sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
3253

33-
# Replace the container name with a test-specific name
34-
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
35-
sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
36-
sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
37-
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
54+
if [[ "$IMAGE_REPO" != "" ]]; then
55+
# Replace the container name with a test-specific name
56+
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
57+
sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
58+
sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
59+
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
60+
fi
61+
3862
# Start Docker Containers
3963
docker compose -f docker_compose.yaml up -d
4064
n=0
@@ -188,12 +212,12 @@ function stop_docker() {
188212
function main() {
189213

190214
stop_docker
191-
192-
begin_time=$(date +%s)
215+
if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
216+
start_time=$(date +%s)
193217
start_services
194218
end_time=$(date +%s)
195-
maximal_duration=$((end_time-begin_time))
196-
echo "Mega service start duration is "$maximal_duration"s" && sleep 1s
219+
duration=$((end_time-start_time))
220+
echo "Mega service start duration is $duration s" && sleep 1s
197221

198222
validate_microservices
199223
validate_megaservice

‎CodeGen/tests/test_codegen_on_gaudi.sh

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,24 @@ WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010

11+
function build_docker_images() {
12+
cd $WORKPATH
13+
git clone https://github.com/opea-project/GenAIComps.git
14+
cd GenAIComps
15+
16+
docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
17+
18+
docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
19+
20+
cd $WORKPATH/docker
21+
docker build --no-cache -t opea/codegen:latest -f Dockerfile .
22+
23+
cd $WORKPATH/docker/ui
24+
docker build --no-cache -t opea/codegen-ui:latest -f docker/Dockerfile .
25+
26+
docker images
27+
}
28+
1129
function start_services() {
1230
cd $WORKPATH/docker/gaudi
1331

@@ -20,11 +38,14 @@ function start_services() {
2038

2139
sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
2240

23-
# Replace the container name with a test-specific name
24-
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
25-
sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
26-
sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
27-
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
41+
if [[ "$IMAGE_REPO" != "" ]]; then
42+
# Replace the container name with a test-specific name
43+
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
44+
sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
45+
sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
46+
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
47+
fi
48+
2849
# Start Docker Containers
2950
docker compose -f docker_compose.yaml up -d
3051

@@ -127,6 +148,7 @@ function main() {
127148

128149
stop_docker
129150

151+
if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
130152
start_services
131153

132154
validate_microservices

‎CodeGen/tests/test_codegen_on_xeon.sh

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,22 @@ WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010

11+
function build_docker_images() {
12+
cd $WORKPATH
13+
git clone https://github.com/opea-project/GenAIComps.git
14+
cd GenAIComps
15+
16+
docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
17+
18+
cd $WORKPATH/docker
19+
docker build --no-cache -t opea/codegen:latest -f Dockerfile .
20+
21+
cd $WORKPATH/docker/ui
22+
docker build --no-cache -t opea/codegen-ui:latest -f docker/Dockerfile .
23+
24+
docker images
25+
}
26+
1127
function start_services() {
1228
cd $WORKPATH/docker/xeon
1329

@@ -20,11 +36,14 @@ function start_services() {
2036

2137
sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
2238

23-
# Replace the container name with a test-specific name
24-
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
25-
sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
26-
sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
27-
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
39+
if [[ "$IMAGE_REPO" != "" ]]; then
40+
# Replace the container name with a test-specific name
41+
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
42+
sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
43+
sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
44+
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
45+
fi
46+
2847
# Start Docker Containers
2948
docker compose -f docker_compose.yaml up -d
3049

@@ -128,6 +147,7 @@ function main() {
128147

129148
stop_docker
130149

150+
if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
131151
start_services
132152

133153
validate_microservices

‎CodeTrans/tests/test_codetrans_on_gaudi.sh

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,14 @@ function start_services() {
4040

4141
sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
4242

43-
# Replace the container name with a test-specific name
44-
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
45-
sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
46-
sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
47-
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
43+
if [[ "$IMAGE_REPO" != "" ]]; then
44+
# Replace the container name with a test-specific name
45+
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
46+
sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
47+
sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
48+
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
49+
fi
50+
4851
# Start Docker Containers
4952
docker compose -f docker_compose.yaml up -d
5053

@@ -145,7 +148,7 @@ function main() {
145148

146149
stop_docker
147150

148-
# build_docker_images
151+
if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
149152
start_services
150153

151154
validate_microservices

‎CodeTrans/tests/test_codetrans_on_xeon.sh

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,22 @@ WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010

11+
function build_docker_images() {
12+
cd $WORKPATH
13+
git clone https://github.com/opea-project/GenAIComps.git
14+
cd GenAIComps
15+
16+
docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
17+
18+
cd $WORKPATH/docker
19+
docker build --no-cache -t opea/codetrans:latest -f Dockerfile .
20+
21+
cd $WORKPATH/docker/ui
22+
docker build --no-cache -t opea/codetrans-ui:latest -f docker/Dockerfile .
23+
24+
docker images
25+
}
26+
1127
function start_services() {
1228
cd $WORKPATH/docker/xeon
1329
export http_proxy=${http_proxy}
@@ -21,11 +37,14 @@ function start_services() {
2137

2238
sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
2339

24-
# Replace the container name with a test-specific name
25-
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
26-
sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
27-
sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
28-
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
40+
if [[ "$IMAGE_REPO" != "" ]]; then
41+
# Replace the container name with a test-specific name
42+
echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
43+
sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
44+
sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
45+
sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
46+
fi
47+
2948
# Start Docker Containers
3049
docker compose -f docker_compose.yaml up -d
3150

@@ -128,6 +147,7 @@ function main() {
128147

129148
stop_docker
130149

150+
if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
131151
start_services
132152

133153
validate_microservices

0 commit comments

Comments
 (0)