naive WIP

Author: rudolfix

.github/workflows/test_common.yml

@@ -31,7 +31,7 @@ jobs:
             python-version: "3.9"
             shell: bash
           - os: ubuntu-latest
-            python-version: "3.11"
+            python-version: "3.10"
             shell: bash
           - os: ubuntu-latest
             python-version: "3.11"

dlt/common/data_types/type_helpers.py

@@ -3,23 +3,25 @@
 import dataclasses
 import datetime  # noqa: I251
 from collections.abc import Mapping as C_Mapping, Sequence as C_Sequence
+from functools import lru_cache
 from typing import Any, Type, Union
 from enum import Enum
 
 from dlt.common.json import custom_pua_remove, json
 from dlt.common.json._simplejson import custom_encode as json_custom_encode
+from dlt.common.typing import copy_sig
 from dlt.common.wei import Wei
 from dlt.common.arithmetics import InvalidOperation, Decimal
 from dlt.common.data_types.typing import TDataType
 from dlt.common.time import (
-    ensure_pendulum_datetime,
+    ensure_pendulum_datetime_non_utc,
     ensure_pendulum_date,
     ensure_pendulum_time,
 )
 from dlt.common.utils import map_nested_in_place, str2bool
 
 
-def py_type_to_sc_type(t: Type[Any]) -> TDataType:
+def _py_type_to_sc_type(t: Type[Any]) -> TDataType:
     # start with most popular types
     if t is str:
         return "text"
@@ -68,14 +70,18 @@ def py_type_to_sc_type(t: Type[Any]) -> TDataType:
     raise TypeError(t)
 
 
+# preserves original signature which lru_cache destroys
+py_type_to_sc_type = copy_sig(_py_type_to_sc_type)(lru_cache(maxsize=None)(_py_type_to_sc_type))
+
+
 def json_to_str(value: Any) -> str:
     return json.dumps(map_nested_in_place(custom_pua_remove, value))
 
 
 def coerce_from_date_types(
     to_type: TDataType, value: datetime.datetime
 ) -> Union[datetime.datetime, datetime.date, datetime.time, int, float, str]:
-    v = ensure_pendulum_datetime(value)
+    v = ensure_pendulum_datetime_non_utc(value)
     if to_type == "timestamp":
         return v
     if to_type == "text":
@@ -179,7 +185,7 @@ def coerce_value(to_type: TDataType, from_type: TDataType, value: Any) -> Any:
 
     try:
         if to_type == "timestamp":
-            return ensure_pendulum_datetime(value)
+            return ensure_pendulum_datetime_non_utc(value)
 
         if to_type == "date":
             return ensure_pendulum_date(value)

dlt/common/destination/capabilities.py

@@ -172,6 +172,8 @@ class DestinationCapabilitiesContext(ContainerInjectableContext):
     supports_truncate_command: bool = True
     schema_supports_numeric_precision: bool = True
     timestamp_precision: int = 6
+    """Default precision of the timestamp type"""
+
     max_rows_per_insert: Optional[int] = None
     insert_values_writer_type: str = "default"
     supports_multiple_statements: bool = True
@@ -209,6 +211,9 @@ class DestinationCapabilitiesContext(ContainerInjectableContext):
     parquet_format: Optional[ParquetFormatConfiguration] = None
     """Parquet format preferred by this destination"""
 
+    supports_naive_datetime: bool = True
+    """The destination can store datetime without timezone"""
+
     def generates_case_sensitive_identifiers(self) -> bool:
         """Tells if capabilities as currently adjusted, will generate case sensitive identifiers"""
         # must have case sensitive support and folding function must preserve casing

dlt/common/pendulum.py

@@ -1,9 +1,6 @@
 from datetime import timedelta, timezone  # noqa: I251
 import pendulum  # noqa: I251
 
-# force UTC as the local timezone to prevent local dates to be written to dbs
-pendulum.set_local_timezone(pendulum.timezone("UTC"))
-
 
 def __utcnow() -> pendulum.DateTime:
     """

dlt/common/pipeline.py

@@ -52,7 +52,7 @@
 )
 from dlt.common.storages.load_package import ParsedLoadJobFileName
 from dlt.common.storages.load_storage import LoadPackageInfo
-from dlt.common.time import ensure_pendulum_datetime, precise_time
+from dlt.common.time import ensure_pendulum_datetime_utc, precise_time
 from dlt.common.typing import DictStrAny, StrAny, SupportsHumanize, TColumnNames
 from dlt.common.data_writers.writers import TLoaderFileFormat
 from dlt.common.utils import RowCounts, merge_row_counts
@@ -413,8 +413,8 @@ def _step_info_complete_load_id(self, load_id: str, metrics: TStepMetrics) -> No
             f"Current load id mismatch {self._current_load_id} != {load_id} when completing step"
             " info"
         )
-        metrics["started_at"] = ensure_pendulum_datetime(self._current_load_started)
-        metrics["finished_at"] = ensure_pendulum_datetime(precise_time())
+        metrics["started_at"] = ensure_pendulum_datetime_utc(self._current_load_started)
+        metrics["finished_at"] = ensure_pendulum_datetime_utc(precise_time())
         self._load_id_metrics[load_id].append(metrics)
         self._current_load_id = None
         self._current_load_started = None