Merge pull request #226 from stac-utils/feat/telemetry

[WIP] feat: add additional logging and telemetry option in application

Author: vincentsarago

CHANGES.md

@@ -1,5 +1,9 @@
 # Release Notes
 
+## Unreleased 
+
+* update titiler requirement to `>=0.23,<0.24`
+
 ## 1.8.0 (2025-05-12)
 
 * update titiler requirement to `>=0.22,<0.23`

pyproject.toml

@@ -28,8 +28,8 @@ classifiers = [
     "Topic :: Scientific/Engineering :: GIS",
 ]
 dependencies = [
-    "titiler.core>=0.22,<0.23",
-    "titiler.mosaic>=0.22,<0.23",
+    "titiler.core>=0.23,<0.24",
+    "titiler.mosaic>=0.23,<0.24",
     "pydantic>=2.4,<3.0",
     "pydantic-settings~=2.0",
 ]
@@ -47,6 +47,13 @@ psycopg-c = [  # C implementation of the libpq wrapper
 psycopg-binary = [  # pre-compiled C implementation
     "psycopg[binary,pool]"
 ]
+telemetry = [
+    "opentelemetry-api",
+    "opentelemetry-sdk",
+    "opentelemetry-instrumentation-fastapi",
+    "opentelemetry-instrumentation-logging",
+    "opentelemetry-exporter-otlp",
+]
 dev = [
     "pre-commit",
     "bump-my-version",

titiler/pgstac/backend.py

@@ -1,6 +1,7 @@
 """TiTiler.PgSTAC custom Mosaic Backend and Custom STACReader."""
 
 import json
+import logging
 from threading import Lock
 from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type
 
@@ -32,6 +33,8 @@
 pgstac_config = PgstacSettings()
 retry_config = RetrySettings()
 
+logger = logging.getLogger(__name__)
+
 
 def multi_points_pgstac(
     asset_list: Sequence[Dict[str, Any]],
@@ -230,7 +233,10 @@ def get_assets(
                     else:
                         raise e
 
-        return resp.get("features", [])
+        features = resp.get("features", [])
+
+        logger.info(f"found {len(features)} assets")
+        return features
 
     @property
     def _quadkeys(self) -> List[str]:
@@ -271,6 +277,9 @@ def _reader(
             with self.reader(item, tms=self.tms, **self.reader_options) as src_dst:
                 return src_dst.tile(x, y, z, **kwargs)
 
+        logger.info(
+            f"reading mosaic of {len(mosaic_assets)} assets with reader: {self.reader}"
+        )
         img, used_assets = mosaic_reader(
             mosaic_assets, _reader, tile_x, tile_y, tile_z, **kwargs
         )
@@ -315,6 +324,7 @@ def _reader(
         if "allowed_exceptions" not in kwargs:
             kwargs.update({"allowed_exceptions": (PointOutsideBounds,)})
 
+        logger.info(f"reading asset values for points with reader: {self.reader}")
         return list(
             multi_points_pgstac(mosaic_assets, _reader, lon, lat, **kwargs).items()
         )
@@ -354,6 +364,9 @@ def _reader(item: Dict[str, Any], bbox: BBox, **kwargs: Any) -> ImageData:
             with self.reader(item, **self.reader_options) as src_dst:
                 return src_dst.part(bbox, **kwargs)
 
+        logger.info(
+            f"reading mosaic of {len(mosaic_assets)} assets with reader: {self.reader}"
+        )
         img, used_assets = mosaic_reader(
             mosaic_assets,
             _reader,
@@ -402,6 +415,9 @@ def _reader(item: Dict[str, Any], shape: Dict, **kwargs: Any) -> ImageData:
             with self.reader(item, **self.reader_options) as src_dst:
                 return src_dst.feature(shape, **kwargs)
 
+        logger.info(
+            f"reading mosaic of {len(mosaic_assets)} assets with reader: {self.reader}"
+        )
         img, used_assets = mosaic_reader(
             mosaic_assets,
             _reader,

titiler/pgstac/factory.py

@@ -1,5 +1,6 @@
 """Custom MosaicTiler Factory for PgSTAC Mosaic Backend."""
 
+import logging
 import os
 import re
 import warnings
@@ -67,6 +68,8 @@
     "yes",
 ]
 
+logger = logging.getLogger(__name__)
+
 
 def _first_value(values: List[Any], default: Any = None):
     """Return the first not None value."""
@@ -117,14 +120,14 @@ class MosaicTilerFactory(BaseFactory):
     conforms_to: Set[str] = field(
         factory=lambda: {
             # https://docs.ogc.org/is/20-057/20-057.html#toc30
-            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/tileset",
+            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset",
             # https://docs.ogc.org/is/20-057/20-057.html#toc34
-            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/tilesets-list",
+            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list",
             # https://docs.ogc.org/is/20-057/20-057.html#toc65
-            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/core",
-            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/png",
-            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/jpeg",
-            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/tiff",
+            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core",
+            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/png",
+            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/jpeg",
+            "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tiff",
         }
     )
 
@@ -196,6 +199,7 @@ def tilejson(
             render_params=Depends(self.render_dependency),
         ):
             """Return TileJSON document for a search_id."""
+            logger.info(f"fetching search info for search {search_id}")
             with request.app.state.dbpool.connection() as conn:
                 with conn.cursor(row_factory=class_row(model.Search)) as cursor:
                     cursor.execute(
@@ -292,6 +296,7 @@ def wmts(  # noqa: C901
             search_id=Depends(self.path_dependency),
         ):
             """OGC WMTS endpoint."""
+            logger.info(f"fetching search info for search {search_id}")
             with request.app.state.dbpool.connection() as conn:
                 with conn.cursor(row_factory=class_row(model.Search)) as cursor:
                     cursor.execute(
@@ -379,6 +384,16 @@ def wmts(  # noqa: C901
                             stacklevel=2,
                         )
 
+            bbox_crs_type = "WGS84BoundingBox"
+            bbox_crs_uri = "urn:ogc:def:crs:OGC:2:84"
+            if tms.rasterio_geographic_crs != WGS84_CRS:
+                bbox_crs_type = "BoundingBox"
+                bbox_crs_uri = CRS_to_urn(tms.rasterio_geographic_crs)
+                # WGS88BoundingBox is always xy ordered, but BoundingBox must match the CRS order
+                if crs_axis_inverted(tms.geographic_crs):
+                    # match the bounding box coordinate order to the CRS
+                    bounds = [bounds[1], bounds[0], bounds[3], bounds[2]]
+
             # LAYER from query-parameters
             qs_key_to_remove = [
                 "tilematrixsetid",
@@ -419,16 +434,6 @@ def wmts(  # noqa: C901
                     "Could not find any valid layers in metadata or construct one from Query Parameters."
                 )
 
-            bbox_crs_type = "WGS84BoundingBox"
-            bbox_crs_uri = "urn:ogc:def:crs:OGC:2:84"
-            if tms.rasterio_geographic_crs != WGS84_CRS:
-                bbox_crs_type = "BoundingBox"
-                bbox_crs_uri = CRS_to_urn(tms.rasterio_geographic_crs)
-                # WGS88BoundingBox is always xy ordered, but BoundingBox must match the CRS order
-                if crs_axis_inverted(tms.geographic_crs):
-                    # match the bounding box coordinate order to the CRS
-                    bounds = [bounds[1], bounds[0], bounds[3], bounds[2]]
-
             return self.templates.TemplateResponse(
                 request,
                 name="wmts.xml",
@@ -486,15 +491,19 @@ def geojson_statistics(
                 fc = FeatureCollection(type="FeatureCollection", features=[geojson])
 
             with rasterio.Env(**env):
+                logger.info(
+                    f"opening data with backend: {self.backend} and reader {self.dataset_reader}"
+                )
                 with self.backend(
                     search_id,
                     reader=self.dataset_reader,
                     reader_options=reader_params.as_dict(),
                     **backend_params.as_dict(),
                 ) as src_dst:
-                    for feature in fc.features:
+                    for i, feature in enumerate(fc.features):
                         shape = feature.model_dump(exclude_none=True)
 
+                        logger.info(f"{i}: reading data")
                         image, _ = src_dst.feature(
                             shape,
                             shape_crs=coord_crs or WGS84_CRS,
@@ -514,8 +523,10 @@ def geojson_statistics(
                         )
 
                         if post_process:
+                            logger.info(f"{i}: post processing image")
                             image = post_process(image)
 
+                        logger.info(f"{i}: calculating statistics")
                         stats = image.statistics(
                             **stats_params.as_dict(),
                             hist_options=histogram_params.as_dict(),
@@ -569,6 +580,9 @@ def bbox_image(
         ):
             """Create image from a bbox."""
             with rasterio.Env(**env):
+                logger.info(
+                    f"opening data with backend: {self.backend} and reader {self.dataset_reader}"
+                )
                 with self.backend(
                     search_id,
                     reader=self.dataset_reader,
@@ -588,6 +602,7 @@ def bbox_image(
                     dst_colormap = getattr(src_dst, "colormap", None)
 
             if post_process:
+                logger.info("post processing image")
                 image = post_process(image)
 
             content, media_type = self.render_func(
@@ -643,6 +658,9 @@ def feature_image(
         ):
             """Create image from a geojson feature."""
             with rasterio.Env(**env):
+                logger.info(
+                    f"opening data with backend: {self.backend} and reader {self.dataset_reader}"
+                )
                 with self.backend(
                     search_id,
                     reader=self.dataset_reader,
@@ -662,6 +680,7 @@ def feature_image(
                     )
 
             if post_process:
+                logger.info("post processing image")
                 image = post_process(image)
 
             content, media_type = self.render_func(

titiler/pgstac/main.py

@@ -123,6 +123,8 @@ async def lifespan(app: FastAPI):
 
 optional_headers = []
 if settings.debug:
+    logging.getLogger("titiler").setLevel(logging.INFO)
+
     app.add_middleware(TotalTimeMiddleware)
     app.add_middleware(LoggerMiddleware)
 
@@ -167,6 +169,33 @@ def pgstac_info(request: Request) -> Dict:
         }
 
 
+if settings.telemetry_enabled:
+    from opentelemetry import trace
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+    from opentelemetry.instrumentation.logging import LoggingInstrumentor
+    from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_VERSION, Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+    LoggingInstrumentor().instrument(set_logging_format=True)
+    FastAPIInstrumentor.instrument_app(app)
+
+    resource = Resource.create(
+        {
+            SERVICE_NAME: "titiler.pgstac",
+            SERVICE_VERSION: titiler_pgstac_version,
+        }
+    )
+
+    provider = TracerProvider(resource=resource)
+
+    # uses the OTEL_EXPORTER_OTLP_ENDPOINT env var
+    processor = BatchSpanProcessor(OTLPSpanExporter())
+    provider.add_span_processor(processor)
+
+    trace.set_tracer_provider(provider)
+
 TITILER_CONFORMS_TO = {
     "http://www.opengis.net/spec/ogcapi-common-1/1.0/req/core",
     "http://www.opengis.net/spec/ogcapi-common-1/1.0/req/landing-page",

titiler/pgstac/settings.py

@@ -32,6 +32,8 @@ class ApiSettings(BaseSettings):
     enable_assets_endpoints: bool = False
     enable_external_dataset_endpoints: bool = False
 
+    telemetry_enabled: bool = False
+
     model_config = SettingsConfigDict(
         env_prefix="TITILER_PGSTAC_API_", env_file=".env", extra="ignore"
     )