Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/test_common.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ jobs:
python-version: "3.11"
shell: bash
uv_sync_args: '--resolution lowest-direct' # could also be 'direct'

# linux test with newest available allowed packages (will update lockfile, should not be committed if run locally)
- os: ubuntu-latest
python-version: "3.11"
shell: bash
uv_sync_args: '--upgrade' # could also be 'direct'

# windows tests
- os: windows-latest
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/test_tools_dashboard.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,18 @@ jobs:
python-version: "3.13"
shell: bash

# linux test with minimal dependencies
- os: ubuntu-latest
python-version: "3.11"
shell: bash
uv_sync_args: '--resolution lowest-direct' # could also be 'direct'

# linux test with newest available allowed packages (will update lockfile, should not be committed if run locally)
- os: ubuntu-latest
python-version: "3.11"
shell: bash
uv_sync_args: '--upgrade' # could also be 'direct'


# windows tests
- os: windows-latest
Expand Down
14 changes: 12 additions & 2 deletions dlt/common/configuration/providers/toml.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,14 @@ def write_toml(self) -> None:

def set_value(self, key: str, value: Any, pipeline_name: Optional[str], *sections: str) -> None:
# write both into tomlkit and dict representations
ConvertError = (
tomlkit.items.ConvertError
if hasattr(tomlkit.items, "ConvertError")
else tomlkit.items._ConvertError
)
try:
self._set_value(self._config_toml, key, value, pipeline_name, *sections)
except tomlkit.items._ConvertError:
except ConvertError:
pass
if hasattr(value, "unwrap"):
value = value.unwrap()
Expand All @@ -114,11 +119,16 @@ def set_fragment(
self, key: Optional[str], value_or_fragment: str, pipeline_name: str, *sections: str
) -> None:
# write both into tomlkit and dict representations
ConvertError = (
tomlkit.items.ConvertError
if hasattr(tomlkit.items, "ConvertError")
else tomlkit.items._ConvertError
)
try:
self._config_toml = self._set_fragment(
self._config_toml, key, value_or_fragment, pipeline_name, *sections
)
except tomlkit.items._ConvertError:
except ConvertError:
pass
super().set_fragment(key, value_or_fragment, pipeline_name, *sections)

Expand Down
4 changes: 2 additions & 2 deletions dlt/helpers/dashboard/ui_elements.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ def build_error_callout(message: str, code: str = None, traceback_string: str =

stack_items = [mo.md(message)]
if code:
stack_items.append(mo.ui.code_editor(code, language="shell"))
stack_items.append(mo.ui.code_editor(code, language="sh"))
if traceback_string:
stack_items.append(
mo.accordion(
{
"Show stacktrace for more information or debugging": mo.ui.code_editor(
traceback_string, language="shell"
traceback_string, language="sh"
)
}
)
Expand Down
6 changes: 1 addition & 5 deletions dlt/helpers/dashboard/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -694,11 +694,7 @@ def build_exception_section(p: dlt.Pipeline) -> List[Any]:

_result.append(
mo.accordion(
{
"Show full stacktrace": mo.ui.code_editor(
"".join(_exception_traces), language="shell"
)
},
{"Show full stacktrace": mo.ui.code_editor("".join(_exception_traces), language="sh")},
lazy=True,
)
)
Expand Down
2 changes: 1 addition & 1 deletion tests/common/configuration/test_toml_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,7 @@ def loader() -> Dict[str, Any]:
provider = CustomLoaderDocProvider("yaml", loader, True)
assert provider.name == "yaml"
assert provider.supports_secrets is True
assert provider.to_toml().startswith("[destination]")
assert provider.to_toml().startswith("[destination")
assert provider.to_yaml().startswith("destination:")
value, _ = provider.get_value("datetime", datetime.datetime, None, "data_types")
assert value == pendulum.parse("1979-05-27 07:32:00-08:00")
Expand Down
7 changes: 4 additions & 3 deletions tests/common/schema/test_coercion.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ def test_coerce_type_to_text() -> None:
# bytes to text (base64)
assert coerce_value("text", "binary", b"binary string") == "YmluYXJ5IHN0cmluZw=="
# HexBytes to text (hex with prefix)
assert (
coerce_value("text", "binary", HexBytes(b"binary string")) == "0x62696e61727920737472696e67"
)
assert coerce_value("text", "binary", HexBytes(b"binary string")) in [
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A newer version of hexbytes returns the hexcode without "0x". We have a couple of options:

  • Keep as is, this means users will see other values in their destination after upgrading this lib
  • Normalize result in normalizer so the result is the same regardless of lib version (kind of a breaking change, either for users on the old version or the new version)
  • Update min version of hexbytes to the version with the new output. Also a kind of breaking change.

Imho this is probably an edge case. I think I would probably keep it as is.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

HexBytes was used to make sure that 0x is present... so if that got removed it does not make sense to use it. My take: write our own HexBytes that alwasy prefixes and parses the prefix. it should be pretty simple to do.

for now I'd merge this ticket

"0x62696e61727920737472696e67",
"62696e61727920737472696e67",
]

# Str enum value
class StrEnum(Enum):
Expand Down