Skip to content

Commit d289b4e

Browse files
yongfengduyongfengdu
authored
Add milvus support for data-prep and retriever-usvc (#468)
Use "helm install" with -f milvus-values.yaml Signed-off-by: Dolpher Du <[email protected]>
1 parent 7b8c510 commit d289b4e

File tree

18 files changed

+157
-12
lines changed

18 files changed

+157
-12
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
**/bin/
22
*.out
3+
*.swp
34
**/Chart.lock
45
**/charts/*.tgz
56

67
bazel-*
78
compile_commands.json
9+
.gitconfig

‎helm-charts/common/data-prep/Chart.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,7 @@ dependencies:
1717
version: 1.0.0
1818
repository: file://../redis-vector-db
1919
condition: redis-vector-db.enabled
20+
- name: milvus
21+
version: 4.2.12
22+
repository: https://zilliztech.github.io/milvus-helm/
23+
condition: milvus.enabled

‎helm-charts/common/data-prep/README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,7 @@ curl http://localhost:6007/v1/dataprep \
5252
| service.port | string | `"6007"` | |
5353
| REDIS_URL | string | `""` | |
5454
| TEI_EMBEDDING_ENDPOINT | string | `""` | |
55+
56+
## Milvus support
57+
58+
Refer to the milvus-values.yaml for milvus configurations.

‎helm-charts/common/data-prep/ci-values.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,5 @@ tei:
99
enabled: true
1010
redis-vector-db:
1111
enabled: true
12+
milvus:
13+
enabled: false

‎helm-charts/common/data-prep/milvus-values.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# Default values for data-prep.
5+
# This is a YAML-formatted file.
6+
# Declare variables to be passed into your templates.
7+
milvus:
8+
enabled: true
9+
cluster:
10+
enabled: false
11+
etcd:
12+
replicaCount: 1
13+
pulsar:
14+
enabled: false
15+
minio:
16+
mode: standalone
17+
redis-vector-db:
18+
enabled: false
19+
tei:
20+
enabled: true
21+
22+
image:
23+
repository: opea/dataprep-milvus
24+
25+
port: 6010
26+
# text embedding inference service URL, e.g. http://<service-name>:<port>
27+
#TEI_EMBEDDING_ENDPOINT: "http://embedding-tei:80"
28+
# milvus DB configurations
29+
#MILVUS_HOST: "milvustest"
30+
MILVUS_PORT: "19530"
31+
COLLECTION_NAME: "rag_milvus"
32+
MOSEC_EMBEDDING_ENDPOINT: ""
33+
MOSEC_EMBEDDING_MODEL: ""

‎helm-charts/common/data-prep/templates/configmap.yaml

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,19 @@ metadata:
88
labels:
99
{{- include "data-prep.labels" . | nindent 4 }}
1010
data:
11-
{{- if .Values.TEI_EMBEDDING_ENDPOINT }}
11+
{{- if .Values.MOSEC_EMBEDDING_ENDPOINT }}
12+
MOSEC_EMBEDDING_ENDPOINT: {{ .Values.MOSEC_EMBEDDING_ENDPOINT | quote}}
13+
MOSEC_EMBEDDING_MODEL: {{ .Values.MOSEC_EMBEDDING_MODEL | quote}}
14+
{{- else if .Values.TEI_EMBEDDING_ENDPOINT }}
1215
TEI_ENDPOINT: {{ .Values.TEI_EMBEDDING_ENDPOINT | quote}}
13-
{{- else if not .Values.EMBED_MODEL }}
16+
TEI_EMBEDDING_ENDPOINT: {{ .Values.TEI_EMBEDDING_ENDPOINT | quote}}
17+
{{- else if not .Values.LOCAL_EMBEDDING_MODEL }}
1418
TEI_ENDPOINT: "http://{{ .Release.Name }}-tei"
1519
{{- end }}
16-
EMBED_MODEL: {{ .Values.EMBED_MODEL | quote }}
20+
{{- if .Values.LOCAL_EMBEDDING_MODEL }}
21+
EMBED_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | quote }}
22+
LOCAL_EMBEDDING_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | quote }}
23+
{{- end }}
1724
{{- if .Values.REDIS_URL }}
1825
REDIS_URL: {{ .Values.REDIS_URL | quote}}
1926
{{- else }}
@@ -22,6 +29,16 @@ data:
2229
INDEX_NAME: {{ .Values.INDEX_NAME | quote }}
2330
KEY_INDEX_NAME: {{ .Values.KEY_INDEX_NAME | quote }}
2431
SEARCH_BATCH_SIZE: {{ .Values.SEARCH_BATCH_SIZE | quote }}
32+
{{- if .Values.MILVUS_HOST }}
33+
MILVUS_HOST: {{ .Values.MILVUS_HOST | quote }}
34+
{{- else }}
35+
MILVUS_HOST: "{{ .Release.Name }}-milvus"
36+
{{- end }}
37+
MILVUS: {{ .Values.MILVUS_HOST | quote }}
38+
MILVUS_PORT: {{ .Values.MILVUS_PORT | quote }}
39+
{{- if .Values.COLLECTION_NAME }}
40+
COLLECTION_NAME: {{ .Values.COLLECTION_NAME | quote }}
41+
{{- end }}
2542
HUGGINGFACEHUB_API_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}}
2643
HF_HOME: "/tmp/.cache/huggingface"
2744
{{- if .Values.global.HF_ENDPOINT }}

‎helm-charts/common/data-prep/templates/deployment.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ spec:
4343
imagePullPolicy: {{ .Values.image.pullPolicy }}
4444
ports:
4545
- name: data-prep
46-
containerPort: 6007
46+
containerPort: {{ .Values.port }}
4747
protocol: TCP
4848
volumeMounts:
4949
- mountPath: /tmp

‎helm-charts/common/data-prep/templates/service.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ spec:
1111
type: {{ .Values.service.type }}
1212
ports:
1313
- port: {{ .Values.service.port }}
14-
targetPort: 6007
14+
targetPort: {{ .Values.port }}
1515
protocol: TCP
1616
name: data-prep
1717
selector:

‎helm-charts/common/data-prep/templates/tests/test-pod.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,9 @@ spec:
2727
curlcode=$?
2828
if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
2929
done;
30+
curl http://{{ include "data-prep.fullname" . }}:{{ .Values.service.port }}/v1/dataprep/delete_file -sS \
31+
-X POST \
32+
-H "Content-Type: application/json" \
33+
-d '{"file_path": "file1.txt"}';
3034
if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
3135
restartPolicy: Never

‎helm-charts/common/data-prep/values.yaml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
tei:
99
enabled: false
10+
milvus:
11+
enabled: false
1012
redis-vector-db:
1113
enabled: false
1214

@@ -38,6 +40,7 @@ securityContext:
3840
seccompProfile:
3941
type: RuntimeDefault
4042

43+
port: 6007
4144
service:
4245
type: ClusterIP
4346
port: 6007
@@ -89,14 +92,21 @@ LOGFLAG: ""
8992
TEI_EMBEDDING_ENDPOINT: ""
9093

9194
# local embedder's model
92-
EMBED_MODEL: ""
95+
LOCAL_EMBEDDING_MODEL: ""
9396

9497
# redis DB service URL, e.g. redis://<service-name>:<port>
9598
REDIS_URL: ""
9699
INDEX_NAME: "rag-redis"
97100
KEY_INDEX_NAME: "file-keys"
98101
SEARCH_BATCH_SIZE: 10
99102

103+
# milvus DB configurations
104+
MILVUS_HOST: ""
105+
MILVUS_PORT: ""
106+
COLLECTION_NAME: ""
107+
MOSEC_EMBEDDING_ENDPOINT: ""
108+
MOSEC_EMBEDDING_MODEL: ""
109+
100110
global:
101111
http_proxy: ""
102112
https_proxy: ""

0 commit comments

Comments
 (0)