Merge pull request #372 from Data-Simply/feature/parameterize-integration-tests

feat: parameterize the integration test cases to reduce the redundancy

Author: mayurkmmt

.github/workflows/bigquery-integration.yml

@@ -60,5 +60,8 @@ jobs:
           TEST_SUITE: ${{ inputs.test_suite }}
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
         run: |
-          uv run pytest tests/integration/bigquery -v \
-            $(if [ "$TEST_SUITE" != "all" ]; then echo "-k $TEST_SUITE"; fi)
+          if [ "$TEST_SUITE" == "all" ]; then
+            uv run pytest tests/integration -k "bigquery" -v
+          else
+            uv run pytest tests/integration -k "bigquery and $TEST_SUITE" -v
+          fi

.github/workflows/pyspark-integration.yml

@@ -38,19 +38,26 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
 
-      - name: Build PySpark Test Image
+      - name: Install uv Package
         run: |
-          docker build -f tests/integration/pyspark/Dockerfile -t pyspark-it .
+          pip install --upgrade pip
+          pip install uv==0.5.30
+
+      - name: Install Dependencies
+        run: |
+          uv sync
 
       - name: Run Integration Tests
         env:
           TEST_SUITE: ${{ inputs.test_suite }}
         run: |
           if [ "$TEST_SUITE" == "all" ]; then
-            docker run --rm pyspark-it
+            uv run pytest tests/integration -k "pyspark" -v
           else
-            docker run --rm pyspark-it uv run pytest -v tests/integration/pyspark/test_${TEST_SUITE}.py
+            uv run pytest tests/integration -k "pyspark and $TEST_SUITE" -v
           fi

.github/workflows/release.yml

@@ -63,7 +63,7 @@ jobs:
         env:
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
         run: |
-          uv run pytest tests/integration/bigquery -v
+          uv run pytest tests/integration -k "bigquery" -v
 
   pyspark-integration-tests:
     name: PySpark Integration Tests
@@ -78,7 +78,7 @@ jobs:
           docker build -f tests/integration/pyspark/Dockerfile -t pyspark-it .
       - name: Run PySpark Integration Tests
         run: |
-          docker run --rm pyspark-it
+          docker run --rm pyspark-it uv run pytest tests/integration -k "pyspark" -v
 
   build:
     name: Build

README.md

@@ -246,8 +246,8 @@ The PySpark integration tests run in a Docker container with a pre-configured Sp
 # Build the PySpark test image
 docker build -f tests/integration/pyspark/Dockerfile -t pyspark-it .
 
-# Run the integration tests
-docker run --rm pyspark-it uv run pytest -v tests/integration/pyspark/test_segstats_segmentation.py
+# Run all PySpark tests
+docker run --rm pyspark-it uv run pytest tests/integration -k "pyspark" -v
 ```
 
 #### BigQuery Integration Tests
@@ -271,8 +271,8 @@ export GCP_PROJECT_ID=your-project-id
 # Install dependencies
 uv sync
 
-# Run all tests
-uv run pytest tests/integration/bigquery -v
+# Run all BigQuery tests
+uv run pytest tests/integration -k "bigquery" -v
 
 # Run specific test module
 uv run pytest tests/integration/bigquery/test_cohort_analysis.py -v