Skip to content

PoC: Tap 14 #2114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 40 commits into from
Closed

PoC: Tap 14 #2114

Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
ce67fe4
Made the sample test repository
abs007 Jun 15, 2022
c8c3bb5
Updated spec versions inside 2.0.0
abs007 Jun 15, 2022
2d830a0
Updated the directory structure
abs007 Jun 17, 2022
1c507c3
Reverted to 6e20c31
abs007 Jun 23, 2022
ffbf4c4
Added the metadata under the newly created TAP 14 folder
abs007 Jun 23, 2022
66b52f8
Updated test_updater_ng.py file
abs007 Jun 26, 2022
99a454e
Updated the test_updater_ng.py file
abs007 Jul 4, 2022
3b2e29b
Updated updater.py
abs007 Jul 6, 2022
f95fdb5
New client updation logic along with tests. Also some test file changes
abs007 Aug 7, 2022
8567160
Updated test functionality and code structure
abs007 Aug 24, 2022
a70cc48
Updated updater.py
abs007 Aug 29, 2022
04231c1
Worked on reviews
abs007 Sep 2, 2022
e3fda3d
Used the black formatter on updater.py
abs007 Sep 2, 2022
5814a2c
Some more changes to updater.py and test_updater_ng.py
abs007 Sep 9, 2022
cf3d41c
Rename TAP 14 directory 1.0.0 -> 1
mnm678 Sep 22, 2022
e0ddd4a
TAP 14 fix tests
mnm678 Sep 22, 2022
3c7be68
fix lint
mnm678 Sep 22, 2022
78f5bfc
fix isort
mnm678 Sep 22, 2022
d3b1493
fix style issues
mnm678 Sep 22, 2022
d65640b
fix lint warnings
mnm678 Sep 22, 2022
250740d
Add TAP 14 integration tests
mnm678 Sep 22, 2022
1e7da4c
Update supported-versions.json to match TAP
mnm678 Sep 22, 2022
2842d05
add tap 14 root update order
mnm678 Sep 22, 2022
08b0c23
fix lint
mnm678 Sep 22, 2022
fa75d78
fix lint
mnm678 Sep 22, 2022
fd3d4d7
fix pylint errors
mnm678 Sep 22, 2022
2d6847c
lint
mnm678 Sep 22, 2022
851b2d6
clean up tests
mnm678 Sep 23, 2022
76567c6
simplify and refactor spec version checks
mnm678 Sep 27, 2022
515b4f2
Move repository supported-versions to root
mnm678 Oct 3, 2022
df2ecd4
Fix typs
mnm678 Oct 3, 2022
6b7cf5d
list -> List for python < 3.9
mnm678 Oct 3, 2022
f57bac7
fix pylint errors
mnm678 Oct 3, 2022
4ffba95
lint fix
mnm678 Oct 21, 2022
81c9718
use supported_versions fields for root update
mnm678 Oct 21, 2022
db67863
format tests
mnm678 Oct 21, 2022
4b5bf7e
remove flag not supported in python < 3.8
mnm678 Oct 24, 2022
a2e58e2
clarify partition
mnm678 Oct 24, 2022
e77f033
Add feature to supported versions
mnm678 Oct 31, 2022
d491d46
fix lint
mnm678 Oct 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion tests/repository_simulator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,10 +205,15 @@ def _fetch(self, url: str) -> Iterator[bytes]:
if path.startswith("/metadata/") and path.endswith(".json"):
# figure out rolename and version
ver_and_name = path[len("/metadata/") :][: -len(".json")]
# inside a version folder
if "/" in ver_and_name:
ver_and_name = ver_and_name.split("/")[1]
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens if there's multiple nested folders? ver_and_name = "/foo/bar.json"

I like

>>> "foo/bar/baz.json".partition("/")
('foo', '/', 'bar/baz.json')
>>> "foo/bar/baz.json".rpartition("/") 
('foo/bar', '/', 'baz.json')

For making intent a little clearer there.

You can also assert len(ver_and_name.split("/") <= 2

version_str, _, role = ver_and_name.partition(".")
# root is always version-prefixed while timestamp is always NOT
if role == Root.type or (
self.root.consistent_snapshot and ver_and_name != Timestamp.type
self.root.consistent_snapshot
and ver_and_name != Timestamp.type
and ver_and_name != "supported-versions"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This isn't a thing anymore, right?

):
version: Optional[int] = int(version_str)
else:
Expand Down
2 changes: 2 additions & 0 deletions tests/test_updater_consistent_snapshot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ def _assert_targets_files_exist(self, filenames: Iterable[str]) -> None:
"consistent_snaphot disabled": {
"consistent_snapshot": False,
"calls": [
("supported-versions", None),
("root", 3),
("timestamp", None),
("snapshot", None),
Expand All @@ -113,6 +114,7 @@ def _assert_targets_files_exist(self, filenames: Iterable[str]) -> None:
"consistent_snaphot enabled": {
"consistent_snapshot": True,
"calls": [
("supported-versions", None),
("root", 3),
("timestamp", None),
("snapshot", 1),
Expand Down
38 changes: 29 additions & 9 deletions tests/test_updater_delegation_graphs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ def _assert_files_exist(self, roles: Iterable[str]) -> None:
"""Assert that local metadata files exist for 'roles'"""
expected_files = sorted([f"{role}.json" for role in roles])
local_metadata_files = sorted(os.listdir(self.metadata_dir))
self.assertListEqual(local_metadata_files, expected_files)
self.assertEqual(expected_files, local_metadata_files)


class TestDelegationsGraphs(TestDelegations):
Expand Down Expand Up @@ -263,7 +263,11 @@ def test_graph_traversal(self, test_data: DelegationsTestCase) -> None:
in the delegator's metadata, using pre-order depth-first search"""

try:
exp_files = [*TOP_LEVEL_ROLE_NAMES, *test_data.visited_order]
exp_files = [
*TOP_LEVEL_ROLE_NAMES,
*test_data.visited_order,
"spec_version",
]
exp_calls = [(role, 1) for role in test_data.visited_order]

self._init_repo(test_data)
Expand All @@ -276,7 +280,7 @@ def test_graph_traversal(self, test_data: DelegationsTestCase) -> None:
updater.refresh()
self.sim.fetch_tracker.metadata.clear()
# Check that metadata dir contains only top-level roles
self._assert_files_exist(TOP_LEVEL_ROLE_NAMES)
self._assert_files_exist([*TOP_LEVEL_ROLE_NAMES, "spec_version"])

# Looking for a non-existing targetpath forces updater
# to visit all possible delegated roles
Expand Down Expand Up @@ -311,7 +315,11 @@ def test_invalid_metadata(self, test_data: DelegationsTestCase) -> None:

self.setup_subtest()
# The invalid role metadata must not be persisted
exp_files = [*TOP_LEVEL_ROLE_NAMES, *test_data.visited_order[:-1]]
exp_files = [
*TOP_LEVEL_ROLE_NAMES,
*test_data.visited_order[:-1],
"spec_version",
]
exp_calls = [(role, 1) for role in test_data.visited_order]

updater = self._init_updater()
Expand Down Expand Up @@ -397,7 +405,11 @@ def test_hash_bins_graph_traversal(
they correctly reffer to the corresponding hash bin prefixes"""

try:
exp_files = [*TOP_LEVEL_ROLE_NAMES, *test_data.visited_order]
exp_files = [
*TOP_LEVEL_ROLE_NAMES,
*test_data.visited_order,
"spec_version",
]
exp_calls = [(role, 1) for role in test_data.visited_order]

self._init_repo(test_data)
Expand All @@ -408,7 +420,7 @@ def test_hash_bins_graph_traversal(
updater.refresh()
self.sim.fetch_tracker.metadata.clear()
# Check that metadata dir contains only top-level roles
self._assert_files_exist(TOP_LEVEL_ROLE_NAMES)
self._assert_files_exist([*TOP_LEVEL_ROLE_NAMES, "spec_version"])

# Looking for a non-existing targetpath forces updater
# to visit a correspondning delegated role
Expand Down Expand Up @@ -481,7 +493,11 @@ def test_succinct_roles_graph_traversal(
# bin should exist locally and only one bin must be downloaded.

try:
exp_files = [*TOP_LEVEL_ROLE_NAMES, test_data.expected_target_bin]
exp_files = [
*TOP_LEVEL_ROLE_NAMES,
test_data.expected_target_bin,
"spec_version",
]
exp_calls = [(test_data.expected_target_bin, 1)]

self.sim = RepositorySimulator()
Expand All @@ -495,7 +511,7 @@ def test_succinct_roles_graph_traversal(
updater.refresh()
self.sim.fetch_tracker.metadata.clear()
# Check that metadata dir contains only top-level roles
self._assert_files_exist(TOP_LEVEL_ROLE_NAMES)
self._assert_files_exist([*TOP_LEVEL_ROLE_NAMES, "spec_version"])

# Looking for a non-existing targetpath forces updater
# to visit a corresponding delegated role.
Expand Down Expand Up @@ -564,7 +580,11 @@ def setUp(self) -> None:
def test_targetfile_search(self, test_data: TargetTestCase) -> None:
try:
self.setup_subtest()
exp_files = [*TOP_LEVEL_ROLE_NAMES, *test_data.visited_order]
exp_files = [
*TOP_LEVEL_ROLE_NAMES,
*test_data.visited_order,
"spec_version",
]
exp_calls = [(role, 1) for role in test_data.visited_order]
exp_target = self.sim.target_files[test_data.targetpath].target_file

Expand Down
Loading