add api/v1 route

add page size
add more tests

Author: sh-rp

app/const.py

@@ -1,2 +1,2 @@
-PAGE_SIZE = 100
-BASE_URL = "http://localhost:8000"
+DEFAULT_PAGE_SIZE = 100
+API_V1_PREFIX = "/api/v1"

app/db.py

@@ -1,7 +1,7 @@
 import duckdb
 import logging
 
-logger = logging.getLogger("fastapi")
+logger = logging.getLogger()
 
 
 POPULATED_DB: duckdb.DuckDBPyConnection | None = None
@@ -27,30 +27,18 @@ def get_db() -> duckdb.DuckDBPyConnection:
         )
         POPULATED_DB = db
 
-        logger.info("Loaded seed data")
-
-        logger.info(
+        logger.warning("Loaded seed data")
+        logger.warning(
             f"Customers: {db.sql('SELECT COUNT(*) FROM customers').fetchone()[0]}"
         )
-        logger.info(db.sql("DESCRIBE customers").fetchall())
-
-        logger.info(f"Orders: {db.sql('SELECT COUNT(*) FROM orders').fetchone()[0]}")
-        logger.info(db.sql("DESCRIBE orders").fetchall())
-
-        logger.info(f"Items: {db.sql('SELECT COUNT(*) FROM items').fetchone()[0]}")
-        logger.info(db.sql("DESCRIBE items").fetchall())
-
-        logger.info(
+        logger.warning(f"Orders: {db.sql('SELECT COUNT(*) FROM orders').fetchone()[0]}")
+        logger.warning(f"Items: {db.sql('SELECT COUNT(*) FROM items').fetchone()[0]}")
+        logger.warning(
             f"Products: {db.sql('SELECT COUNT(*) FROM products').fetchone()[0]}"
         )
-        logger.info(db.sql("DESCRIBE products").fetchall())
-
-        logger.info(f"Stores: {db.sql('SELECT COUNT(*) FROM stores').fetchone()[0]}")
-        logger.info(db.sql("DESCRIBE stores").fetchall())
-
-        logger.info(
+        logger.warning(f"Stores: {db.sql('SELECT COUNT(*) FROM stores').fetchone()[0]}")
+        logger.warning(
             f"Supplies: {db.sql('SELECT COUNT(*) FROM supplies').fetchone()[0]}"
         )
-        logger.info(db.sql("DESCRIBE supplies").fetchall())
 
     return POPULATED_DB

app/main.py

@@ -1,15 +1,17 @@
 from fastapi import FastAPI
 
 from app import routers
+from app.const import API_V1_PREFIX
 
 app = FastAPI()
 
-app.include_router(routers.customers_router)
-app.include_router(routers.orders_router)
-app.include_router(routers.item_router)
-app.include_router(routers.product_router)
-app.include_router(routers.supplies_router)
-app.include_router(routers.store_router)
+# api v1
+app.include_router(routers.customers_router, prefix=API_V1_PREFIX)
+app.include_router(routers.orders_router, prefix=API_V1_PREFIX)
+app.include_router(routers.item_router, prefix=API_V1_PREFIX)
+app.include_router(routers.product_router, prefix=API_V1_PREFIX)
+app.include_router(routers.supplies_router, prefix=API_V1_PREFIX)
+app.include_router(routers.store_router, prefix=API_V1_PREFIX)
 
 
 @app.get("/")

app/models.py

@@ -0,0 +1,10 @@
+"""
+TODO: add pydantic models for jaffle shop
+"""
+
+from pydantic import BaseModel
+
+
+class Customer(BaseModel):
+    id: str
+    name: str

app/routers.py

@@ -1,15 +1,17 @@
 """
-All jaffle shop routers
+All jaffle shop routers for api v1
 """
 
+from typing import List
+
 import duckdb
 from urllib.parse import urlencode
 from fastapi import APIRouter, Depends, Response, Request
 
 from app.db import get_db
-from app.const import PAGE_SIZE
+from app.const import DEFAULT_PAGE_SIZE, API_V1_PREFIX
+from app.models import Customer
 
-# defined routers
 customers_router = APIRouter(tags=["customers"])
 orders_router = APIRouter(tags=["orders"])
 item_router = APIRouter(tags=["items"])
@@ -35,14 +37,15 @@ def _get_paged_response(
     response: Response = None,
     where_clause: str = "",
     sort_by: str = "",
+    page_size: int = DEFAULT_PAGE_SIZE,
 ):
     """
     Get a paged response from a collection endpoint
     Will insert next header if applicable
     """
 
-    offset = (page - 1) * PAGE_SIZE
-    last_item = offset + PAGE_SIZE
+    offset = (page - 1) * page_size
+    last_item = offset + page_size
     count = db.query(f"SELECT COUNT(*) FROM {table_name} {where_clause}").fetchone()[0]
 
     # get base url from request
@@ -51,7 +54,7 @@ def _get_paged_response(
     # )
     # scheme = request.headers.get("X-Forwarded-Proto", "http")
 
-    url = f"/{table_name}"
+    url = API_V1_PREFIX + f"/{table_name}"
     query_params = dict(request.query_params)
     if last_item < count and response:
         query_params["page"] = page + 1
@@ -60,7 +63,7 @@ def _get_paged_response(
 
     return _get_list_response(
         db,
-        f"SELECT * FROM {table_name} {where_clause} {sort_by} LIMIT {PAGE_SIZE} OFFSET {offset}",
+        f"SELECT * FROM {table_name} {where_clause} {sort_by} LIMIT {page_size} OFFSET {offset}",
     )
 
 
@@ -78,17 +81,25 @@ def _get_single_response(db: duckdb.DuckDBPyConnection, query: str):
 #
 
 
-@customers_router.get("/customers")
+@customers_router.get("/customers", response_model=List[Customer])
 async def get_customers(
     page: int = 1,
+    page_size: int = DEFAULT_PAGE_SIZE,
     response: Response = None,
     request: Request = None,
     db: duckdb.DuckDBPyConnection = Depends(get_db),
 ):
-    return _get_paged_response(db, request, "customers", page, response)
+    return _get_paged_response(
+        db=db,
+        request=request,
+        table_name="customers",
+        page=page,
+        response=response,
+        page_size=page_size,
+    )
 
 
-@customers_router.get("/customers/{customer_id}")
+@customers_router.get("/customers/{customer_id}", response_model=Customer)
 async def get_customer(
     customer_id: str, db: duckdb.DuckDBPyConnection = Depends(get_db)
 ):
@@ -116,6 +127,7 @@ def _enrich_orders(db: duckdb.DuckDBPyConnection, orders: list[dict]):
 @orders_router.get("/orders")
 async def get_orders(
     page: int = 1,
+    page_size: int = DEFAULT_PAGE_SIZE,
     start_date: str = None,
     end_date: str = None,
     response: Response = None,
@@ -124,14 +136,19 @@ async def get_orders(
 ):
     where_clause = ""
     if start_date:
-        where_clause += f" AND ordered_at >= '{start_date}'"
+        where_clause += f" AND ordered_at::DATE >= '{start_date}'"
     if end_date:
-        where_clause += f" AND ordered_at <= '{end_date}'"
+        where_clause += f" AND ordered_at::DATE <= '{end_date}'"
     if where_clause:
         where_clause = f"WHERE {where_clause[5:]}"
-    sort_by = "ORDER BY ordered_at ASC"
     orders = _get_paged_response(
-        db, request, "orders", page, response, where_clause, sort_by
+        db=db,
+        request=request,
+        table_name="orders",
+        page=page,
+        page_size=page_size,
+        response=response,
+        where_clause=where_clause,
     )
     return _enrich_orders(db, orders)
 
@@ -147,11 +164,19 @@ async def get_order(order_id: str, db: duckdb.DuckDBPyConnection = Depends(get_d
 @item_router.get("/items")
 async def get_items(
     page: int = 1,
+    page_size: int = DEFAULT_PAGE_SIZE,
     response: Response = None,
     request: Request = None,
     db: duckdb.DuckDBPyConnection = Depends(get_db),
 ):
-    return _get_paged_response(db, request, "items", page, response)
+    return _get_paged_response(
+        db=db,
+        request=request,
+        table_name="items",
+        page=page,
+        response=response,
+        page_size=page_size,
+    )
 
 
 @item_router.get("/items/{item_id}")
@@ -165,11 +190,19 @@ async def get_item(item_id: str, db: duckdb.DuckDBPyConnection = Depends(get_db)
 @product_router.get("/products")
 async def get_products(
     page: int = 1,
+    page_size: int = DEFAULT_PAGE_SIZE,
     response: Response = None,
     request: Request = None,
     db: duckdb.DuckDBPyConnection = Depends(get_db),
 ):
-    return _get_paged_response(db, request, "products", page, response)
+    return _get_paged_response(
+        db=db,
+        request=request,
+        table_name="products",
+        page=page,
+        response=response,
+        page_size=page_size,
+    )
 
 
 @product_router.get("/products/{sku}")
@@ -183,11 +216,19 @@ async def get_product(sku: str, db: duckdb.DuckDBPyConnection = Depends(get_db))
 @store_router.get("/stores")
 async def get_stores(
     page: int = 1,
+    page_size: int = DEFAULT_PAGE_SIZE,
     response: Response = None,
     request: Request = None,
     db: duckdb.DuckDBPyConnection = Depends(get_db),
 ):
-    return _get_paged_response(db, request, "stores", page, response)
+    return _get_paged_response(
+        db=db,
+        request=request,
+        table_name="stores",
+        page=page,
+        response=response,
+        page_size=page_size,
+    )
 
 
 @store_router.get("/stores/{store_id}")
@@ -201,11 +242,19 @@ async def get_store(store_id: str, db: duckdb.DuckDBPyConnection = Depends(get_d
 @supplies_router.get("/supplies")
 async def get_supplies(
     page: int = 1,
+    page_size: int = DEFAULT_PAGE_SIZE,
     response: Response = None,
     request: Request = None,
     db: duckdb.DuckDBPyConnection = Depends(get_db),
 ):
-    return _get_paged_response(db, request, "supplies", page, response)
+    return _get_paged_response(
+        db=db,
+        request=request,
+        table_name="supplies",
+        page=page,
+        response=response,
+        page_size=page_size,
+    )
 
 
 @supplies_router.get("/supplies/{supply_id}")

tests/test_dlt_extraction.py

@@ -15,6 +15,7 @@
 from dlt.sources.rest_api import rest_api_source
 from dlt.common.destination.dataset import SupportsReadableDataset
 
+from tests.utils import EXPECTED_TABLES_COUNTS_JANUARY_2017
 
 app = TestClient(app)
 
@@ -44,7 +45,7 @@ def test_extract_full_dataset():
     source = rest_api_source(
         {
             "client": {
-                "base_url": "http://localhost:8000",
+                "base_url": "http://localhost:8000/api/v1",
                 "paginator": {
                     "type": "header_link",
                 },
@@ -62,6 +63,7 @@ def test_extract_full_dataset():
                     "endpoint": {
                         "path": "orders",
                         "params": {
+                            "page_size": 1000,
                             "start_date": "2017-01-01",
                             "end_date": "2017-01-31",
                         },
@@ -79,14 +81,7 @@ def test_extract_full_dataset():
     )
 
     pipeline.run(source)
-    assert row_counts(pipeline.dataset()) == {
-        "customers": 935,
-        "products": 10,
-        "stores": 6,
-        "supplies": 65,
-        "orders": 3303,  # 3303 orders in january 2017
-        "orders__items": 5072,
-    }
+    assert row_counts(pipeline.dataset()) == EXPECTED_TABLES_COUNTS_JANUARY_2017
 
 
 @pytest.mark.skip(reason="This is a live test against the deployed shop")