changes all configspecs to conform with new init methods, drops special init for credentials

Author: rudolfix

dlt/common/__init__.py

@@ -1,8 +1,8 @@
+from dlt.common import logger
 from dlt.common.arithmetics import Decimal
 from dlt.common.wei import Wei
 from dlt.common.pendulum import pendulum
 from dlt.common.json import json
 from dlt.common.runtime.signals import sleep
-from dlt.common.runtime import logger
 
 __all__ = ["Decimal", "Wei", "pendulum", "json", "sleep", "logger"]

dlt/common/configuration/resolve.py

@@ -76,7 +76,7 @@ def initialize_credentials(hint: Any, initial_value: Any) -> CredentialsConfigur
         first_credentials: CredentialsConfiguration = None
         for idx, spec in enumerate(specs_in_union):
             try:
-                credentials = spec(initial_value)
+                credentials = spec.from_init_value(initial_value)
                 if credentials.is_resolved():
                     return credentials
                 # keep first credentials in the union to return in case all of the match but not resolve
@@ -88,7 +88,7 @@ def initialize_credentials(hint: Any, initial_value: Any) -> CredentialsConfigur
         return first_credentials
     else:
         assert issubclass(hint, CredentialsConfiguration)
-        return hint(initial_value)  # type: ignore
+        return hint.from_init_value(initial_value)  # type: ignore
 
 
 def inject_section(

dlt/common/configuration/specs/api_credentials.py

@@ -6,9 +6,9 @@
 
 @configspec
 class OAuth2Credentials(CredentialsConfiguration):
-    client_id: str
-    client_secret: TSecretValue
-    refresh_token: Optional[TSecretValue]
+    client_id: str = None
+    client_secret: TSecretValue = None
+    refresh_token: Optional[TSecretValue] = None
     scopes: Optional[List[str]] = None
 
     token: Optional[TSecretValue] = None

dlt/common/configuration/specs/aws_credentials.py

@@ -121,3 +121,9 @@ def parse_native_representation(self, native_value: Any) -> None:
                 self.__is_resolved__ = True
         except Exception:
             raise InvalidBoto3Session(self.__class__, native_value)
+
+    @classmethod
+    def from_session(cls, botocore_session: Any) -> "AwsCredentials":
+        self = cls()
+        self.parse_native_representation(botocore_session)
+        return self

dlt/common/configuration/specs/config_providers_context.py

@@ -1,4 +1,5 @@
 import contextlib
+import dataclasses
 import io
 from typing import ClassVar, List
 
@@ -28,7 +29,7 @@ class ConfigProvidersConfiguration(BaseConfiguration):
     only_toml_fragments: bool = True
 
     # always look in providers
-    __section__ = known_sections.PROVIDERS
+    __section__: ClassVar[str] = known_sections.PROVIDERS
 
 
 @configspec
@@ -37,8 +38,12 @@ class ConfigProvidersContext(ContainerInjectableContext):
 
     global_affinity: ClassVar[bool] = True
 
-    providers: List[ConfigProvider]
-    context_provider: ConfigProvider
+    providers: List[ConfigProvider] = dataclasses.field(
+        default=None, init=False, repr=False, compare=False
+    )
+    context_provider: ConfigProvider = dataclasses.field(
+        default=None, init=False, repr=False, compare=False
+    )
 
     def __init__(self) -> None:
         super().__init__()

dlt/common/configuration/specs/config_section_context.py

@@ -8,7 +8,7 @@
 class ConfigSectionContext(ContainerInjectableContext):
     TMergeFunc = Callable[["ConfigSectionContext", "ConfigSectionContext"], None]
 
-    pipeline_name: Optional[str]
+    pipeline_name: Optional[str] = None
     sections: Tuple[str, ...] = ()
     merge_style: TMergeFunc = None
     source_state_key: str = None
@@ -70,13 +70,3 @@ def __str__(self) -> str:
             super().__str__()
             + f": {self.pipeline_name} {self.sections}@{self.merge_style} state['{self.source_state_key}']"
         )
-
-    if TYPE_CHECKING:
-        # provide __init__ signature when type checking
-        def __init__(
-            self,
-            pipeline_name: str = None,
-            sections: Tuple[str, ...] = (),
-            merge_style: TMergeFunc = None,
-            source_state_key: str = None,
-        ) -> None: ...

dlt/common/configuration/specs/connection_string_credentials.py

@@ -1,14 +1,15 @@
-from typing import Any, ClassVar, Dict, List, Optional
+import dataclasses
+from typing import Any, ClassVar, Dict, List, Optional, Union
+
 from dlt.common.libs.sql_alchemy import URL, make_url
 from dlt.common.configuration.specs.exceptions import InvalidConnectionString
-
 from dlt.common.typing import TSecretValue
 from dlt.common.configuration.specs.base_configuration import CredentialsConfiguration, configspec
 
 
 @configspec
 class ConnectionStringCredentials(CredentialsConfiguration):
-    drivername: str = None
+    drivername: str = dataclasses.field(default=None, init=False, repr=False, compare=False)
     database: str = None
     password: Optional[TSecretValue] = None
     username: str = None
@@ -18,6 +19,11 @@ class ConnectionStringCredentials(CredentialsConfiguration):
 
     __config_gen_annotations__: ClassVar[List[str]] = ["port", "password", "host"]
 
+    def __init__(self, connection_string: Union[str, Dict[str, Any]] = None) -> None:
+        """Initializes the credentials from SQLAlchemy like connection string or from dict holding connection string elements"""
+        super().__init__()
+        self._apply_init_value(connection_string)
+
     def parse_native_representation(self, native_value: Any) -> None:
         if not isinstance(native_value, str):
             raise InvalidConnectionString(self.__class__, native_value, self.drivername)

dlt/common/configuration/specs/gcp_credentials.py

@@ -1,5 +1,6 @@
+import dataclasses
 import sys
-from typing import Any, Final, List, Tuple, Union, Dict
+from typing import Any, ClassVar, Final, List, Tuple, Union, Dict
 
 from dlt.common import json, pendulum
 from dlt.common.configuration.specs.api_credentials import OAuth2Credentials
@@ -22,8 +23,12 @@
 
 @configspec
 class GcpCredentials(CredentialsConfiguration):
-    token_uri: Final[str] = "https://oauth2.googleapis.com/token"
-    auth_uri: Final[str] = "https://accounts.google.com/o/oauth2/auth"
+    token_uri: Final[str] = dataclasses.field(
+        default="https://oauth2.googleapis.com/token", init=False, repr=False, compare=False
+    )
+    auth_uri: Final[str] = dataclasses.field(
+        default="https://accounts.google.com/o/oauth2/auth", init=False, repr=False, compare=False
+    )
 
     project_id: str = None
 
@@ -69,7 +74,9 @@ def to_gcs_credentials(self) -> Dict[str, Any]:
 class GcpServiceAccountCredentialsWithoutDefaults(GcpCredentials):
     private_key: TSecretValue = None
     client_email: str = None
-    type: Final[str] = "service_account"  # noqa: A003
+    type: Final[str] = dataclasses.field(  # noqa: A003
+        default="service_account", init=False, repr=False, compare=False
+    )
 
     def parse_native_representation(self, native_value: Any) -> None:
         """Accepts ServiceAccountCredentials as native value. In other case reverts to serialized services.json"""
@@ -121,8 +128,10 @@ def __str__(self) -> str:
 @configspec
 class GcpOAuthCredentialsWithoutDefaults(GcpCredentials, OAuth2Credentials):
     # only desktop app supported
-    refresh_token: TSecretValue
-    client_type: Final[str] = "installed"
+    refresh_token: TSecretValue = None
+    client_type: Final[str] = dataclasses.field(
+        default="installed", init=False, repr=False, compare=False
+    )
 
     def parse_native_representation(self, native_value: Any) -> None:
         """Accepts Google OAuth2 credentials as native value. In other case reverts to serialized oauth client secret json"""
@@ -237,7 +246,7 @@ def __str__(self) -> str:
 
 @configspec
 class GcpDefaultCredentials(CredentialsWithDefault, GcpCredentials):
-    _LAST_FAILED_DEFAULT: float = 0.0
+    _LAST_FAILED_DEFAULT: ClassVar[float] = 0.0
 
     def parse_native_representation(self, native_value: Any) -> None:
         """Accepts google credentials as native value"""

dlt/common/configuration/specs/known_sections.py

@@ -13,6 +13,9 @@
 EXTRACT = "extract"
 """extract stage of the pipeline"""
 
+SCHEMA = "schema"
+"""schema configuration, ie. normalizers"""
+
 PROVIDERS = "providers"
 """secrets and config providers"""
 

dlt/common/configuration/specs/run_configuration.py

@@ -1,7 +1,7 @@
 import binascii
 from os.path import isfile, join
 from pathlib import Path
-from typing import Any, Optional, Tuple, IO
+from typing import Any, ClassVar, Optional, IO
 from dlt.common.typing import TSecretStrValue
 
 from dlt.common.utils import encoding_for_mode, main_module_file_path, reveal_pseudo_secret
@@ -30,7 +30,7 @@ class RunConfiguration(BaseConfiguration):
     """Platform connection"""
     dlthub_dsn: Optional[TSecretStrValue] = None
 
-    __section__ = "runtime"
+    __section__: ClassVar[str] = "runtime"
 
     def on_resolved(self) -> None:
         # generate pipeline name from the entry point script name