SWP-302: added api schema

Author: o-zander

requirements.txt

@@ -5,6 +5,7 @@ Django==4.1.13
 django-elasticsearch-dsl==8.0
 django-filter==2.4.0
 djangorestframework==3.12.2
+drf-spectacular==0.29.0
 elasticsearch<9
 pikepdf==5.6.1
 playwright==1.27

swp/api/v1/publicationlist/viewsets.py

@@ -1,3 +1,4 @@
+from drf_spectacular.utils import extend_schema
 from rest_framework.decorators import action
 
 from swp.api.v1.viewsets import SWPViewSet
@@ -18,10 +19,12 @@ class PublicationListViewSet(SWPViewSet):
     def get_queryset(self):
         return self.queryset.filter(user=self.request.user)
 
+    @extend_schema(operation_id='publication_list_add_publication')
     @action(['POST'], detail=True, serializer_class=PublicationListAddSerializer)
     def add(self, request, *args, **kwargs):
         return self.update(request, *args, **kwargs)
 
+    @extend_schema(operation_id='publication_list_remove_publication')
     @action(['POST'], detail=True, serializer_class=PublicationListRemoveSerializer)
     def remove(self, request, *args, **kwargs):
         return self.update(request, *args, **kwargs)

swp/api/v1/scraper/serializers.py

@@ -1,4 +1,5 @@
 from contextlib import suppress
+from typing import Optional
 
 from django.utils.translation import gettext_lazy as _
 
@@ -12,6 +13,7 @@
 from swp.api.v1.serializers import ActivatableSerializer
 from swp.models import Scraper
 from swp.tasks import preview_scraper
+from swp.tasks.scraper import PreviewResult
 
 STATES = [
     PENDING,
@@ -61,6 +63,6 @@ def update(self, instance: Scraper, validated_data):
         return preview_scraper.delay(start_url, data)
 
     @staticmethod
-    def get_result(instance: AsyncResult):
+    def get_result(instance: AsyncResult) -> Optional[PreviewResult]:
         with suppress(TimeoutError):
             return instance.get(timeout=0.1, propagate=False)

swp/api/v1/scraper/viewsets.py

@@ -1,5 +1,6 @@
 from django.db.models import Prefetch
 
+from drf_spectacular.utils import extend_schema
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
@@ -26,10 +27,12 @@ def check_object_permissions(self, request, obj: Scraper):
         if self.action in {'update', 'partial_update'} and obj.is_active:
             raise ScraperActiveException
 
-    @action(['POST'], detail=True, serializer_class=ScraperPreviewSerializer)
+    @extend_schema(operation_id='scraper_preview_start')
+    @action(['PATCH'], detail=True, serializer_class=ScraperPreviewSerializer)
     def preview(self, request, *args, **kwargs):
         return self.partial_update(request, *args, **kwargs)
 
+    @extend_schema(operation_id='scraper_preview_status')
     @action(detail=False, url_path=r'preview/(?P<task_id>[a-z0-9-]+)', serializer_class=ScraperPreviewSerializer)
     def preview_status(self, request, *, task_id: str, **kwargs):
         result = preview_scraper.AsyncResult(task_id)

swp/api/v1/spectacular.py

@@ -0,0 +1,31 @@
+from typing import List
+
+from drf_spectacular.openapi import AutoSchema
+
+from .viewsets import SWPViewSet
+
+
+class SWPSchema(AutoSchema):
+
+    def is_excluded(self) -> bool:
+        return 'v1' not in self.path
+
+    def _tokenize_path(self) -> List[str]:
+        """
+        Remove the v1 prefix from tokenized path.
+        """
+
+        prefix, *tokenized_path = AutoSchema._tokenize_path(self)
+
+        return tokenized_path
+
+    def get_operation_id(self) -> str:
+        operation_id = AutoSchema.get_operation_id(self)
+
+        if isinstance(self.view, SWPViewSet):
+            if self.view.action not in self.method_mapping.values():
+                action = self.method_mapping[self.method.lower()]
+
+                return operation_id.removesuffix(f'_{action}')
+
+        return operation_id

swp/settings/base.py

@@ -43,6 +43,7 @@
     # Extensions
     'django_elasticsearch_dsl',
     'django_filters',
+    'drf_spectacular',
     'rest_framework',
 
     # Admin
@@ -263,6 +264,7 @@
         'rest_framework.permissions.IsAuthenticated',
     ],
     'DEFAULT_RENDERER_CLASSES': REST_FRAMEWORK_DEFAULT_RENDERER_CLASSES,
+    'DEFAULT_SCHEMA_CLASS': 'swp.api.v1.spectacular.SWPSchema',
     'DEFAULT_FILTER_BACKENDS': [
         'django_filters.rest_framework.DjangoFilterBackend',
         'rest_framework.filters.OrderingFilter',
@@ -273,6 +275,10 @@
 
 # </editor-fold>
 
+SPECTACULAR_SETTINGS = {
+    'TITLE': 'WebMonitor',
+}
+
 # <editor-fold desc="Zotero">
 
 ZOTERO_API_BASE_URL = 'https://api.zotero.org'

swp/tasks/scraper.py

@@ -1,3 +1,5 @@
+from typing import List, TypedDict, Union
+
 from asgiref.sync import async_to_sync
 
 from swp.celery import app
@@ -9,6 +11,21 @@
 PUBLICATION_PREVIEW_PAGES = 2
 
 
+class SuccessResult(TypedDict):
+    success: bool
+    publications: List[dict]
+    max_per_page: int
+    is_multipage: bool
+
+
+class ErrorResult(TypedDict):
+    success: bool
+    error: str
+
+
+PreviewResult = Union[SuccessResult, ErrorResult]
+
+
 def configure_preview_pagination(config: dict) -> int:
     """ Setup scraper config for limited pagination during preview. """
     paginator = config.pop('paginator', None) or {}
@@ -21,7 +38,7 @@ def configure_preview_pagination(config: dict) -> int:
     return max_pages * max_per_page
 
 
-async def scrape(scraper: Scraper, config: dict) -> dict:
+async def scrape(scraper: Scraper, config: dict) -> PreviewResult:
     publications = []
 
     max_len = configure_preview_pagination(config)
@@ -57,15 +74,15 @@ async def scrape(scraper: Scraper, config: dict) -> dict:
     }
 
 
-def clean_publications(publications):
+def clean_publications(publications: List[dict]):
     for publication in publications:
         if fields := publication.get('fields'):
             for embedding_field in ['pdf_path', 'text_content']:
                 fields.pop(embedding_field, None)
 
 
 @app.task(name='preview.scraper')
-def preview_scraper(start_url, config):
+def preview_scraper(start_url, config) -> PreviewResult:
     scraper = Scraper(start_url)
 
     result = async_to_sync(scrape)(scraper, config)

swp/urls.py

@@ -2,6 +2,8 @@
 from django.contrib import admin
 from django.urls import include, path, register_converter
 
+from drf_spectacular.views import SpectacularAPIView
+
 from swp.api import default_router as internal
 from swp.api.v1 import default_router as v1
 from swp.converters import DateConverter
@@ -25,6 +27,7 @@
     path('admin/', admin.site.urls),
 
     # api
+    path('api/v1/schema/', SpectacularAPIView.as_view(), name='schema'),
     path('api/v1/', v1.urls),
     path('api/', internal.urls),