Skip to content

Commit 341f97a

Browse files
chensuyuechensuyue
authored
image scan and publish cd enhance (#604)
Signed-off-by: chensuyue <[email protected]>
1 parent 2093558 commit 341f97a

File tree

3 files changed

+157
-30
lines changed

3 files changed

+157
-30
lines changed

‎.github/workflows/_get-image-list.yml

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Get Image List
5+
permissions: read-all
6+
on:
7+
workflow_call:
8+
inputs:
9+
services:
10+
default: ""
11+
required: false
12+
type: string
13+
images:
14+
default: ""
15+
required: false
16+
type: string
17+
mode:
18+
default: "CD"
19+
required: false
20+
type: string
21+
outputs:
22+
matrix:
23+
description: "Image List"
24+
value: ${{ jobs.get-image-list.outputs.matrix }}
25+
26+
jobs:
27+
get-image-list:
28+
runs-on: ubuntu-latest
29+
outputs:
30+
matrix: ${{ steps.get-matrix.outputs.matrix }}
31+
steps:
32+
- name: Checkout out Repo
33+
uses: actions/checkout@v4
34+
35+
- name: Set Matrix
36+
id: get-matrix
37+
run: |
38+
image_list=[]
39+
if [[ ! -z "${{ inputs.services }}" ]]; then
40+
pip install yq
41+
services=($(echo ${{ inputs.services }} | tr ',' ' '))
42+
for service in ${services[@]}
43+
do
44+
if [[ "${{ inputs.mode }}" == "CD" ]]; then
45+
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose-cd.yaml
46+
else
47+
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
48+
fi
49+
if [ -f "$docker_compose_yml" ]; then
50+
images=$(cat $docker_compose_yml | yq -r '.[]' | jq 'keys' | jq -c '.')
51+
image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
52+
fi
53+
done
54+
fi
55+
56+
if [[ ! -z "${{ inputs.images }}" ]]; then
57+
images=($(echo ${{ inputs.images }} | tr ',' ' '))
58+
input_image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.')
59+
image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${input_image_list}))
60+
fi
61+
62+
echo "print image list..."
63+
echo "$image_list" | jq . | jq -r '.[]'
64+
echo "end of image list..."
65+
echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT

‎.github/workflows/manual-docker-publish.yml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Services publish docker image on manual event
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
services:
9+
default: "asr"
10+
description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
11+
required: false
12+
type: string
13+
images:
14+
default: ""
15+
description: "List of images to scan [dataprep-pgvector,dataprep-pinecone...]"
16+
required: false
17+
type: string
18+
tag:
19+
default: "v0.9"
20+
description: "Tag to publish"
21+
required: true
22+
type: string
23+
publish_tags:
24+
default: "latest,v0.9"
25+
description: "Tag list apply to publish images"
26+
required: false
27+
type: string
28+
29+
permissions: read-all
30+
jobs:
31+
get-image-list:
32+
uses: ./.github/workflows/_get-image-list.yml
33+
with:
34+
services: ${{ inputs.services }}
35+
images: ${{ inputs.images }}
36+
mode: ${{ inputs.mode }}
37+
38+
publish:
39+
needs: [get-image-list]
40+
strategy:
41+
matrix:
42+
image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
43+
runs-on: "docker-build-gaudi"
44+
steps:
45+
- uses: docker/[email protected]
46+
with:
47+
username: ${{ secrets.DOCKERHUB_USER }}
48+
password: ${{ secrets.DOCKERHUB_TOKEN }}
49+
50+
- name: Image Publish
51+
uses: opea-project/validation/actions/image-publish@main
52+
with:
53+
local_image_ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
54+
image_name: opea/${{ matrix.image }}
55+
publish_tags: ${{ inputs.publish_tags }}

‎.github/workflows/manual-bom-scan.yml renamed to ‎.github/workflows/manual-docker-scan.yml

Lines changed: 37 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4-
name: Comps docker images BoM scan on manual event
4+
name: Services docker images BoM/CVE scan on manual event
55
on:
66
workflow_dispatch:
77
inputs:
88
services:
99
default: "asr"
1010
description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus...]" #,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
11-
required: true
11+
required: false
12+
type: string
13+
images:
14+
default: ""
15+
description: "List of images to scan [dataprep-pgvector,dataprep-pinecone...]"
16+
required: false
1217
type: string
1318
tag:
1419
default: "comps"
@@ -25,31 +30,22 @@ on:
2530
description: "Enable trivy-scan"
2631
required: false
2732
type: boolean
33+
mode:
34+
default: "CD"
35+
description: "Whether the test range is CI or CD"
36+
required: false
37+
type: string
2838

2939
permissions: read-all
3040
jobs:
3141
get-image-list:
32-
runs-on: ubuntu-latest
33-
outputs:
34-
matrix: ${{ steps.scan-matrix.outputs.matrix }}
35-
steps:
36-
- name: Checkout out Repo
37-
uses: actions/checkout@v4
38-
39-
- name: Set Matrix
40-
id: scan-matrix
41-
run: |
42-
pip install yq
43-
services=($(echo ${{ inputs.services }} | tr ',' ' '))
44-
image_list=[]
45-
for service in ${services[@]}
46-
do
47-
images=$(cat ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.')
48-
image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
49-
done
50-
echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT
42+
uses: ./.github/workflows/_get-image-list.yml
43+
with:
44+
services: ${{ inputs.services }}
45+
images: ${{ inputs.images }}
46+
mode: ${{ inputs.mode }}
5147

52-
scan-license:
48+
scan-docker:
5349
needs: get-image-list
5450
runs-on: "docker-build-gaudi"
5551
strategy:
@@ -59,8 +55,7 @@ jobs:
5955
steps:
6056
- name: Pull Image
6157
run: |
62-
docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:latest
63-
# ${{ inputs.tag }}
58+
docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
6459
echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
6560
6661
- name: SBOM Scan Container
@@ -87,16 +82,28 @@ jobs:
8782
if: always()
8883
run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
8984

90-
- uses: actions/[email protected]
85+
- name: Collect Logs
9186
if: always()
87+
run: |
88+
mkdir -p /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}
89+
mv ${{ matrix.image }}-*-scan.txt /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}
90+
91+
upload-artifacts:
92+
needs: scan-docker
93+
runs-on: "docker-build-gaudi"
94+
if: always()
95+
steps:
96+
- uses: actions/[email protected]
9297
with:
93-
name: sbom-scan
94-
path: ${{ matrix.image }}-sbom-scan.txt
98+
name: sbom-scan-${{ inputs.tag }}-${{ github.run_number }}
99+
path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-sbom-scan.txt
95100
overwrite: true
96101

97102
- uses: actions/[email protected]
98-
if: always()
99103
with:
100-
name: trivy-scan
101-
path: ${{ matrix.image }}-trivy-scan.txt
104+
name: trivy-scan-${{ inputs.tag }}-${{ github.run_number }}
105+
path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-trivy-scan.txt
102106
overwrite: true
107+
108+
- name: Remove Logs
109+
run: rm -rf /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} && rm -rf /tmp/sbom-action-*

0 commit comments

Comments
 (0)