feat(rye): changed package manager (#3019)

Needs export UV_INDEX_STRATEGY=unsafe-first-match to work

Completes CORE-153

Author: StanGirard

.github/workflows/backend-core-tests.yml

@@ -20,20 +20,18 @@ jobs:
           - 9998:9998
 
     steps:
-      - name: Checkout code
+      - name: 👀 Checkout code
         uses: actions/checkout@v2
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - name: 🔨 Install the latest version of rye
+        uses: eifinger/setup-rye@v4
         with:
-          python-version: "3.11"
-
-      - name: Install dependencies
+          enable-cache: true
+          working-directory: backend
+      - name: 🔄 Sync dependencies
         run: |
-          cd backend/core
-          python -m pip install --upgrade pip
-          pip install poetry
-          poetry install -E base --with dev,test
+          cd backend
+          UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
 
       - name: Run tests
         env:
@@ -42,6 +40,5 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr  pandoc
-          tesseract --version
           cd backend/core
-          poetry run tox
+          rye test

.github/workflows/backend-tests.yml

@@ -11,20 +11,22 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout code
+      - name: 👀 Checkout code
         uses: actions/checkout@v2
 
-      - name: Install poetry
-        run: pipx install poetry
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - name: 🔨 Install the latest version of rye
+        uses: eifinger/setup-rye@v4
         with:
-          python-version: "3.11"
-          cache: poetry
+          enable-cache: true
+          working-directory: backend
+
+      - name: 🔄 Sync dependencies
+        run: |
+          cd backend
+          UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
 
 
-      - name: Install Supabase CLI
+      - name: 🚤 Install Supabase CLI
         run: |
           ARCHITECTURE=$(uname -m)
           if [ "$ARCHITECTURE" = "x86_64" ]; then
@@ -35,7 +37,7 @@ jobs:
             sudo dpkg -i supabase_1.163.6_linux_arm64.deb
           fi
 
-      - name: Install system dependencies
+      - name: 😭 Install system dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr pandoc
@@ -55,10 +57,8 @@ jobs:
             CELEBRY_BROKER_QUEUE_NAME: quivr-preview.fifo
             QUIVR_DOMAIN: http://localhost:3000/
             BACKEND_URL: http://localhost:5050
+            EMBEDDING_DIM: 1536
         run: |
-          tesseract --version
           cd backend
-          poetry install --with dev,test
           supabase db start
-          poetry run pytest worker
-          poetry run pytest api
+          rye test -p quivr-api -p quivr-worker

Makefile

@@ -19,13 +19,12 @@ dev-build:
 ## prod: Build and start production environment
 .PHONY: prod
 prod:
-	docker compose build backend-core
 	docker compose -f docker-compose.yml up --build
 
 ## front: Build and start frontend
 .PHONY: front
 front:
-	cd frontend  && yarn build && yarn start
+	cd frontend  && yarn && yarn build && yarn start
 
 ## test: Run tests
 .PHONY: test

backend/.gitignore

@@ -0,0 +1,10 @@
+# python generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# venv
+.venv

backend/Dockerfile

@@ -1,11 +1,9 @@
-# Using a slim version for a smaller base image
 FROM python:3.11.6-slim-bullseye
 
-ARG DEV_MODE
-ENV DEV_MODE=$DEV_MODE
+WORKDIR /app
 
-# Install GEOS library, Rust, and other dependencies, then clean up
-RUN apt-get clean && apt-get update && apt-get install -y \
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \
     libgeos-dev \
     libcurl4-openssl-dev \
     libssl-dev \
@@ -15,23 +13,14 @@ RUN apt-get clean && apt-get update && apt-get install -y \
     git \
     poppler-utils \
     tesseract-ocr \
-    autoconf \
-    automake \
-    build-essential \
-    libtool \
-    python-dev \
-    wget \
-    # Additional dependencies for document handling
     libmagic-dev \
-    poppler-utils \
-    tesseract-ocr \
     libreoffice \
     libpq-dev \
     gcc \
-    pandoc && \
-    rm -rf /var/lib/apt/lists/*
-
+    wget \
+    && rm -rf /var/lib/apt/lists/*
 
+# Install Supabase CLI
 RUN ARCHITECTURE=$(uname -m) && \
     if [ "$ARCHITECTURE" = "x86_64" ]; then \
     wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_amd64.deb && \
@@ -41,43 +30,15 @@ RUN ARCHITECTURE=$(uname -m) && \
     wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_arm64.deb && \
     dpkg -i supabase_1.163.6_linux_arm64.deb && \
     rm supabase_1.163.6_linux_arm64.deb; \
-    fi && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python && \
-    cd /usr/local/bin && \
-    ln -s /opt/poetry/bin/poetry && \
-    poetry config virtualenvs.create false
-
-
-# Add Rust binaries to the PATH
-ENV PATH="/root/.cargo/bin:${PATH}" \
-    POETRY_CACHE_DIR=/tmp/poetry_cache \
-    PYTHONDONTWRITEBYTECODE=1
-
-WORKDIR /code
-
-# Copy monorepo dependencies
-# CORE
-COPY core/pyproject.toml core/README.md core/poetry.lock  /code/core/
-COPY core/quivr_core /code/core/quivr_core
-# API
-COPY api/pyproject.toml  api/poetry.lock api/README.md /code/api/
-COPY api/quivr_api /code/api/quivr_api
-
-COPY ./pyproject.toml ./poetry.lock* /code/
-
-COPY ./ci-migration.sh /code/
+    fi
 
-COPY supabase /code/supabase/
+COPY . .
 
-# Run install
-RUN poetry install --no-root --with dev,test && \
-    playwright install --with-deps && \
-    rm -rf $POETRY_CACHE_DIR
+RUN PYTHONDONTWRITEBYTECODE=1 pip install --no-cache-dir -r requirements.lock
 
+RUN playwright install --with-deps
 
-ENV PYTHONPATH=/code
 
+ENV PYTHONPATH=/app
 
-EXPOSE 5050
+EXPOSE 5050

backend/Dockerfile.dev

@@ -1,9 +1,7 @@
 # Using a slim version for a smaller base image
 FROM python:3.11.6-slim-bullseye
 
-ARG DEV_MODE
-ENV DEV_MODE=$DEV_MODE
-
+WORKDIR /app
 # Install GEOS library, Rust, and other dependencies, then clean up
 RUN apt-get clean && apt-get update && apt-get install -y \
     libgeos-dev \
@@ -28,40 +26,18 @@ RUN apt-get clean && apt-get update && apt-get install -y \
     pandoc && \
     rm -rf /var/lib/apt/lists/* && apt-get clean
 
-# TODO(@aminediro) : multistage build. Probably dont neet poetry once its built
-# Install Poetry
-RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python && \
-    cd /usr/local/bin && \
-    ln -s /opt/poetry/bin/poetry && \
-    poetry config virtualenvs.create false
-
-# Add Rust binaries to the PATH
-ENV PATH="/root/.cargo/bin:${PATH}" \
-    POETRY_CACHE_DIR=/tmp/poetry_cache \
-    PYTHONDONTWRITEBYTECODE=1
-
-WORKDIR /code/worker
-
-# Copy monorepo dependencies
-# CORE
-COPY core/pyproject.toml core/README.md core/poetry.lock  /code/core/
-COPY core/quivr_core /code/core/quivr_core
-# API
-COPY ../api/pyproject.toml  ../api/poetry.lock ../api/README.md /code/api/
-COPY ../api/quivr_api /code/api/quivr_api
-COPY ./pyproject.toml ./poetry.lock* /code/
-
-# Worker
-COPY pyproject.toml  poetry.lock README.md /code/api/
-COPY quivr_api /code/api/quivr_api
-COPY pyproject.toml poetry.lock* /code/
+COPY requirements.lock  pyproject.toml README.md ./
+COPY api/pyproject.toml api/README.md ./api/
+COPY api/quivr_api/__init__.py ./api/quivr_api/__init__.py
+COPY core/pyproject.toml core/README.md ./core/
+COPY core/quivr_core/__init__.py ./core/quivr_core/__init__.py
+COPY worker/pyproject.toml worker/README.md ./worker/
+COPY worker/quivr_worker/__init__.py ./worker/quivr_worker/__init__.py
 
-# Run install
-RUN poetry install --no-root --with dev,test && \
-    playwright install --with-deps && \
-    rm -rf $POETRY_CACHE_DIR
+RUN PYTHONDONTWRITEBYTECODE=1 pip install --no-cache-dir -r requirements.lock
 
+RUN playwright install --with-deps
 
-ENV PYTHONPATH=/code
+COPY . .
 
-EXPOSE 5050
+EXPOSE 5050

backend/README.md

@@ -0,0 +1,3 @@
+# backend
+
+Describe your project here.

backend/api/.gitignore

@@ -0,0 +1,10 @@
+# python generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# venv
+.venv

backend/api/.python-version

@@ -0,0 +1 @@
+3.11.9