Skip to content

Commit 995420c

Browse files
AmineDirochloediaStanGirardZewed
authored
feat: separate celery workers (#2948)
# Description - Complete refacto of workers - New docker image for worker NOTE: Syncs are under heavy refacto ! --------- Co-authored-by: chloedia <[email protected]> Co-authored-by: Stan Girard <[email protected]> Co-authored-by: Antoine Dewez <[email protected]> Co-authored-by: aminediro <[email protected]> Co-authored-by: Stan Girard <[email protected]> Co-authored-by: Chloé Daems <[email protected]>
1 parent 6855585 commit 995420c

File tree

156 files changed

+16761
-15439
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

156 files changed

+16761
-15439
lines changed

‎.github/workflows/backend-core-tests.yml‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ name: Run Tests with Tika Server
33
on:
44
push:
55
paths:
6-
- 'backend/core/**'
6+
- "backend/core/**"
77
pull_request:
88
paths:
9-
- 'backend/core/**'
9+
- "backend/core/**"
1010
workflow_dispatch:
1111

1212
jobs:
@@ -20,25 +20,28 @@ jobs:
2020
- 9998:9998
2121

2222
steps:
23-
- name: Checkout code
24-
uses: actions/checkout@v2
23+
- name: Checkout code
24+
uses: actions/checkout@v2
2525

26-
- name: Set up Python
27-
uses: actions/setup-python@v2
28-
with:
29-
python-version: '3.11'
26+
- name: Set up Python
27+
uses: actions/setup-python@v2
28+
with:
29+
python-version: "3.11"
3030

31-
- name: Install dependencies
32-
run: |
33-
cd backend/core
34-
python -m pip install --upgrade pip
35-
pip install poetry
36-
poetry install -E base
31+
- name: Install dependencies
32+
run: |
33+
cd backend/core
34+
python -m pip install --upgrade pip
35+
pip install poetry
36+
poetry install -E base --with dev,test
3737
38-
- name: Run tests
39-
env:
40-
TIKA_URL: http://localhost:9998/tika
41-
OPENAI_API_KEY: this-is-a-test-key
42-
run: |
43-
cd backend/core
44-
poetry run pytest tests
38+
- name: Run tests
39+
env:
40+
TIKA_URL: http://localhost:9998/tika
41+
OPENAI_API_KEY: this-is-a-test-key
42+
run: |
43+
sudo apt-get update
44+
sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr pandoc
45+
tesseract --version
46+
cd backend/core
47+
poetry run tox

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ backend/core/local_models/*
5959

6060
## scripts
6161
package-lock.json
62-
backend/celerybeat-schedule
62+
celerybeat-schedule
6363
frontend/public/robots.txt
6464
frontend/public/sitemap*
6565

‎.release-please-manifest.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
2-
"backend/core": "0.0.12",
2+
"backend/core": "0.0.13",
33
".": "0.0.293"
44
}

‎backend/.dockerignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,5 @@
1010
**/.docusaurus/
1111
**/node_modules/
1212
**/.venv/
13+
**/.tox/
14+
**/.tox-docker/

‎backend/Dockerfile.dev‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,22 @@ ENV PATH="/root/.cargo/bin:${PATH}" \
4040
POETRY_CACHE_DIR=/tmp/poetry_cache \
4141
PYTHONDONTWRITEBYTECODE=1
4242

43-
WORKDIR /code
43+
WORKDIR /code/worker
4444

4545
# Copy monorepo dependencies
4646
# CORE
4747
COPY core/pyproject.toml core/README.md core/poetry.lock /code/core/
4848
COPY core/quivr_core /code/core/quivr_core
4949
# API
50-
COPY api/pyproject.toml api/poetry.lock api/README.md /code/api/
51-
COPY api/quivr_api /code/api/quivr_api
50+
COPY ../api/pyproject.toml ../api/poetry.lock ../api/README.md /code/api/
51+
COPY ../api/quivr_api /code/api/quivr_api
5252
COPY ./pyproject.toml ./poetry.lock* /code/
5353

54+
# Worker
55+
COPY pyproject.toml poetry.lock README.md /code/api/
56+
COPY quivr_api /code/api/quivr_api
57+
COPY pyproject.toml poetry.lock* /code/
58+
5459
# Run install
5560
RUN poetry install --no-root --with dev,test && \
5661
playwright install --with-deps && \

‎backend/api/Dockerfile‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Using a slim version for a smaller base image
2+
FROM python:3.11.6-slim-bullseye
3+
4+
ARG DEV_MODE
5+
ENV DEV_MODE=$DEV_MODE
6+
7+
RUN apt-get clean && apt-get update && apt-get install -y wget curl
8+
9+
RUN ARCHITECTURE=$(uname -m) && \
10+
if [ "$ARCHITECTURE" = "x86_64" ]; then \
11+
wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_amd64.deb && \
12+
dpkg -i supabase_1.163.6_linux_amd64.deb && \
13+
rm supabase_1.163.6_linux_amd64.deb; \
14+
elif [ "$ARCHITECTURE" = "aarch64" ]; then \
15+
wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_arm64.deb && \
16+
dpkg -i supabase_1.163.6_linux_arm64.deb && \
17+
rm supabase_1.163.6_linux_arm64.deb; \
18+
fi && \
19+
rm -rf /var/lib/apt/lists/*
20+
21+
# TODO(@aminediro) : multistage build. Probably dont neet poetry once its built
22+
# Install Poetry
23+
RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python && \
24+
cd /usr/local/bin && \
25+
ln -s /opt/poetry/bin/poetry && \
26+
poetry config virtualenvs.create false && \
27+
poetry config virtualenvs.in-project false
28+
29+
# Add Rust binaries to the PATH
30+
ENV PATH="/root/.cargo/bin:${PATH}" \
31+
POETRY_CACHE_DIR=/tmp/poetry_cache \
32+
PYTHONDONTWRITEBYTECODE=1 \
33+
POETRY_VIRTUALENVS_PATH=/code/api/.venv-docker
34+
35+
WORKDIR /code/api
36+
COPY . /code/
37+
38+
RUN poetry install && rm -rf $POETRY_CACHE_DIR
39+
40+
ENV PYTHONPATH=/code/api
41+
42+
EXPOSE 5050

‎backend/api/Dockerfile.dev‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Using a slim version for a smaller base image
2+
FROM python:3.11.6-slim-bullseye
3+
4+
ARG DEV_MODE
5+
ENV DEV_MODE=$DEV_MODE
6+
7+
RUN apt-get clean && apt-get update && apt-get install -y wget curl
8+
9+
RUN ARCHITECTURE=$(uname -m) && \
10+
if [ "$ARCHITECTURE" = "x86_64" ]; then \
11+
wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_amd64.deb && \
12+
dpkg -i supabase_1.163.6_linux_amd64.deb && \
13+
rm supabase_1.163.6_linux_amd64.deb; \
14+
elif [ "$ARCHITECTURE" = "aarch64" ]; then \
15+
wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_arm64.deb && \
16+
dpkg -i supabase_1.163.6_linux_arm64.deb && \
17+
rm supabase_1.163.6_linux_arm64.deb; \
18+
fi && \
19+
rm -rf /var/lib/apt/lists/*
20+
21+
# TODO(@aminediro) : multistage build. Probably dont neet poetry once its built
22+
# Install Poetry
23+
RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python && \
24+
cd /usr/local/bin && \
25+
ln -s /opt/poetry/bin/poetry && \
26+
poetry config virtualenvs.create false && \
27+
poetry config virtualenvs.in-project false
28+
29+
# Add Rust binaries to the PATH
30+
ENV PATH="/root/.cargo/bin:${PATH}" \
31+
POETRY_CACHE_DIR=/tmp/poetry_cache \
32+
PYTHONDONTWRITEBYTECODE=1 \
33+
POETRY_VIRTUALENVS_PATH=/code/api/.venv-docker
34+
35+
WORKDIR /code/api
36+
COPY api/pyproject.toml api/poetry.lock api/README.md /code/api/
37+
COPY api/quivr_api /code/api/quivr_api
38+
39+
# Run install
40+
# Run install
41+
RUN poetry install --no-directory --no-root --with dev && rm -rf $POETRY_CACHE_DIR
42+
43+
ENV PYTHONPATH=/code/api
44+
45+
EXPOSE 5050

‎backend/api/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
# Quivr backend
1+
# quivr-api

0 commit comments

Comments
 (0)