Feat/1596 adds custom docs config provider

dlt/cli/init_command.py

@@ -1,6 +1,7 @@
 import os
 import ast
 import shutil
+import tomlkit
 from types import ModuleType
 from typing import Dict, List, Sequence, Tuple
 from importlib.metadata import version as pkg_version
@@ -488,16 +489,19 @@ def init_command(
 
     # generate tomls with comments
     secrets_prov = SecretsTomlProvider()
-    # print(secrets_prov._toml)
-    write_values(secrets_prov._toml, required_secrets.values(), overwrite_existing=False)
+    secrets_toml = tomlkit.document()
+    write_values(secrets_toml, required_secrets.values(), overwrite_existing=False)
+    secrets_prov._config_doc = secrets_toml
+
     config_prov = ConfigTomlProvider()
-    write_values(config_prov._toml, required_config.values(), overwrite_existing=False)
+    config_toml = tomlkit.document()
+    write_values(config_toml, required_config.values(), overwrite_existing=False)
+    config_prov._config_doc = config_toml
+
     # write toml files
     secrets_prov.write_toml()
     config_prov.write_toml()
 
-    # telemetry_status_command()
-
     # if there's no dependency system write the requirements file
     if dependency_system is None:
         requirements_txt = "\n".join(source_files.requirements.compiled())

dlt/cli/telemetry_command.py

@@ -1,6 +1,6 @@
 import os
+import tomlkit
 
-from dlt.common.configuration import resolve_configuration
 from dlt.common.configuration.container import Container
 from dlt.common.configuration.providers.toml import ConfigTomlProvider
 from dlt.common.configuration.specs import RunConfiguration
@@ -29,15 +29,21 @@ def change_telemetry_status_command(enabled: bool) -> None:
     ]
     # write local config
     config = ConfigTomlProvider(add_global_config=False)
+    config_toml = tomlkit.document()
     if not config.is_empty:
-        write_values(config._toml, telemetry_value, overwrite_existing=True)
+        write_values(config_toml, telemetry_value, overwrite_existing=True)
+        config._config_doc = config_toml
         config.write_toml()
+
     # write global config
     global_path = ConfigTomlProvider.global_config_path()
     os.makedirs(global_path, exist_ok=True)
     config = ConfigTomlProvider(project_dir=global_path, add_global_config=False)
-    write_values(config._toml, telemetry_value, overwrite_existing=True)
+    config_toml = tomlkit.document()
+    write_values(config_toml, telemetry_value, overwrite_existing=True)
+    config._config_doc = config_toml
     config.write_toml()
+
     if enabled:
         fmt.echo("Telemetry switched %s" % fmt.bold("ON"))
     else:

dlt/common/configuration/accessors.py

@@ -80,6 +80,13 @@ def _get_value(self, field: str, type_hint: Type[Any] = None) -> Tuple[Any, List
                 break
         return value, traces
 
+    @staticmethod
+    def register_provider(provider: ConfigProvider) -> None:
+        """Registers `provider` to participate in the configuration resolution. `provider`
+        is added after all existing providers and will be used if all others do not resolve.
+        """
+        Container()[ConfigProvidersContext].add_provider(provider)
+
 
 class _ConfigAccessor(_Accessor):
     """Provides direct access to configured values that are not secrets."""

dlt/common/configuration/providers/__init__.py

@@ -4,12 +4,13 @@
 from .toml import (
     SecretsTomlProvider,
     ConfigTomlProvider,
-    TomlFileProvider,
+    ProjectDocProvider,
     CONFIG_TOML,
     SECRETS_TOML,
     StringTomlProvider,
-    SECRETS_TOML_KEY,
+    CustomLoaderDocProvider,
 )
+from .vault import SECRETS_TOML_KEY
 from .google_secrets import GoogleSecretsProvider
 from .context import ContextProvider
 
@@ -19,11 +20,12 @@
     "DictionaryProvider",
     "SecretsTomlProvider",
     "ConfigTomlProvider",
-    "TomlFileProvider",
+    "ProjectDocProvider",
     "CONFIG_TOML",
     "SECRETS_TOML",
     "StringTomlProvider",
     "SECRETS_TOML_KEY",
     "GoogleSecretsProvider",
     "ContextProvider",
+    "CustomLoaderDocProvider",
 ]

dlt/common/configuration/providers/airflow.py

@@ -1,10 +1,10 @@
 import io
 import contextlib
 
-from .toml import VaultTomlProvider
+from .vault import VaultDocProvider
 
 
-class AirflowSecretsTomlProvider(VaultTomlProvider):
+class AirflowSecretsTomlProvider(VaultDocProvider):
     def __init__(self, only_secrets: bool = False, only_toml_fragments: bool = False) -> None:
         super().__init__(only_secrets, only_toml_fragments)
 

dlt/common/configuration/providers/dictionary.py

@@ -1,38 +1,26 @@
 from contextlib import contextmanager
-from typing import Any, ClassVar, Iterator, Optional, Type, Tuple
+from typing import ClassVar, Iterator
 
-from dlt.common.typing import StrAny
+from dlt.common.typing import DictStrAny
 
-from .provider import ConfigProvider, get_key_name
+from .provider import get_key_name
+from .toml import BaseDocProvider
 
 
-class DictionaryProvider(ConfigProvider):
+class DictionaryProvider(BaseDocProvider):
     NAME: ClassVar[str] = "Dictionary Provider"
 
     def __init__(self) -> None:
-        self._values: StrAny = {}
+        super().__init__({})
+
+    @staticmethod
+    def get_key_name(key: str, *sections: str) -> str:
+        return get_key_name(key, "__", *sections)
 
     @property
     def name(self) -> str:
         return self.NAME
 
-    def get_value(
-        self, key: str, hint: Type[Any], pipeline_name: str, *sections: str
-    ) -> Tuple[Optional[Any], str]:
-        full_path = sections + (key,)
-        if pipeline_name:
-            full_path = (pipeline_name,) + full_path
-        full_key = get_key_name(key, "__", pipeline_name, *sections)
-        node = self._values
-        try:
-            for k in full_path:
-                if not isinstance(node, dict):
-                    raise KeyError(k)
-                node = node[k]
-            return node, full_key
-        except KeyError:
-            return None, full_key
-
     @property
     def supports_secrets(self) -> bool:
         return True
@@ -42,8 +30,8 @@ def supports_sections(self) -> bool:
         return True
 
     @contextmanager
-    def values(self, v: StrAny) -> Iterator[None]:
-        p_values = self._values
-        self._values = v
+    def values(self, v: DictStrAny) -> Iterator[None]:
+        p_values = self._config_doc
+        self._config_doc = v
         yield
-        self._values = p_values
+        self._config_doc = p_values

dlt/common/configuration/providers/google_secrets.py

@@ -5,7 +5,7 @@
 from dlt.common.json import json
 from dlt.common.configuration.specs import GcpServiceAccountCredentials
 from dlt.common.exceptions import MissingDependencyException
-from .toml import VaultTomlProvider
+from .vault import VaultDocProvider
 from .provider import get_key_name
 
 # Create a translation table to replace punctuation with ""
@@ -33,7 +33,7 @@ def normalize_key(in_string: str) -> str:
     return stripped_whitespace
 
 
-class GoogleSecretsProvider(VaultTomlProvider):
+class GoogleSecretsProvider(VaultDocProvider):
     def __init__(
         self,
         credentials: GcpServiceAccountCredentials,