From 4d99f78cf9e18880077fbdeea9c7fa9f712f3299 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 23 Nov 2022 13:55:19 +0200 Subject: [PATCH 01/15] Rename manual repository example I plan to add another repository example as well. Signed-off-by: Jussi Kukkonen --- examples/README.md | 3 +-- examples/{repo_example => manual_repo}/basic_repo.py | 0 .../hashed_bin_delegation.py | 0 .../succinct_hash_bin_delegations.py | 0 tests/test_examples.py | 10 +++++----- 5 files changed, 6 insertions(+), 7 deletions(-) rename examples/{repo_example => manual_repo}/basic_repo.py (100%) rename examples/{repo_example => manual_repo}/hashed_bin_delegation.py (100%) rename examples/{repo_example => manual_repo}/succinct_hash_bin_delegations.py (100%) diff --git a/examples/README.md b/examples/README.md index 8ca5d7bf0f..fd1a1db1a1 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,4 @@ # Usage examples * [client](client_example) -* [repository](repo_example) - +* [repository built with low-level Metadata API](manual_repo) diff --git a/examples/repo_example/basic_repo.py b/examples/manual_repo/basic_repo.py similarity index 100% rename from examples/repo_example/basic_repo.py rename to examples/manual_repo/basic_repo.py diff --git a/examples/repo_example/hashed_bin_delegation.py b/examples/manual_repo/hashed_bin_delegation.py similarity index 100% rename from examples/repo_example/hashed_bin_delegation.py rename to examples/manual_repo/hashed_bin_delegation.py diff --git a/examples/repo_example/succinct_hash_bin_delegations.py b/examples/manual_repo/succinct_hash_bin_delegations.py similarity index 100% rename from examples/repo_example/succinct_hash_bin_delegations.py rename to examples/manual_repo/succinct_hash_bin_delegations.py diff --git a/tests/test_examples.py b/tests/test_examples.py index daa8839507..3fd24d03dd 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -17,10 +17,10 @@ class TestRepoExamples(unittest.TestCase): - """Unit test class for 'repo_example' scripts. + """Unit test class for 'manual_repo' scripts. Provides a '_run_example_script' method to run (exec) a script located in - the 'repo_example' directory. + the 'manual_repo' directory. """ @@ -28,9 +28,9 @@ class TestRepoExamples(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - """Locate and cache 'repo_example' dir.""" + """Locate the example dir.""" base = Path(__file__).resolve().parents[1] - cls.repo_examples_dir = base / "examples" / "repo_example" + cls.repo_examples_dir = base / "examples" / "manual_repo" def setUp(self) -> None: """Create and change into test dir. @@ -48,7 +48,7 @@ def tearDown(self) -> None: def _run_script_and_assert_files( self, script_name: str, filenames_created: List[str] ) -> None: - """Run script in 'repo_example' dir and assert that it created the + """Run script in exmple dir and assert that it created the files corresponding to the passed filenames inside a 'tmp*' test dir at CWD.""" script_path = str(self.repo_examples_dir / script_name) From 5e17617fc5c2b9295fa9731e92e082e0329b3a36 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Thu, 24 Nov 2022 17:05:18 +0200 Subject: [PATCH 02/15] Add repository module Plan for tuf.repository is: * provides useful functionality for TUF repository-side implementations (repository applications, developer tools, etc) * is minimalistic: only features that most implementations will use should be icluded * Only example implementations will be provided in python-tuf * As more repository implementations are built using tuf.repository we can evaluate what extended functionality is useful In this PR, a single abstract class is added that provides a framework for building repository-modifying tools. In subsequent commits some examples will be added that demonstrate how to use the class. Signed-off-by: Jussi Kukkonen --- tuf/repository/__init__.py | 6 ++ tuf/repository/_repository.py | 139 ++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 tuf/repository/__init__.py create mode 100644 tuf/repository/_repository.py diff --git a/tuf/repository/__init__.py b/tuf/repository/__init__.py new file mode 100644 index 0000000000..ee28015fce --- /dev/null +++ b/tuf/repository/__init__.py @@ -0,0 +1,6 @@ +# Copyright 2021-2022 python-tuf contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +"""Repository API: A library to help repository implementations""" + +from tuf.repository._repository import AbortEdit, Repository diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py new file mode 100644 index 0000000000..4f1d2b98df --- /dev/null +++ b/tuf/repository/_repository.py @@ -0,0 +1,139 @@ +# Copyright 2021-2022 python-tuf contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +"""Repository Abstraction for metadata management""" + +import logging +from abc import ABC, abstractmethod +from contextlib import contextmanager, suppress +from typing import Dict, Generator, Optional, Tuple + +from tuf.api.metadata import Metadata, MetaFile, Signed + +logger = logging.getLogger(__name__) + + +class AbortEdit(Exception): + """Raise to exit the edit() contextmanager without saving changes""" + + +class Repository(ABC): + """Abstract class for metadata modifying implementations + + This class is intended to be a base class used in any metadata editing + application, whether it is a real repository server or a developer tool. + + Implementations must implement open() and close(), and can then use the + edit() contextmanager to implement actual operations. + + A few operations (sign, snapshot and timestamp) are already implemented + in this base class. + """ + + @abstractmethod + def open(self, role: str, init: bool = False) -> Metadata: + """Load a roles metadata from storage or cache, return it + + If 'init', then create metadata from scratch""" + raise NotImplementedError + + @abstractmethod + def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: + """Write roles metadata into storage + + If sign_only, then just append signatures of all available keys. + + If not sign_only, update expiry and version and replace signatures + with ones from all available keys.""" + raise NotImplementedError + + @contextmanager + def edit( + self, role: str, init: bool = False + ) -> Generator[Signed, None, None]: + """Context manager for editing a roles metadata + + Context manager takes care of loading the roles metadata (or creating + new metadata if 'init'), updating expiry and version. The caller can do + other changes to the Signed object and when the context manager exits, + a new version of the roles metadata is stored. + + Context manager user can raise AbortEdit from inside the with-block to + cancel the edit: in this case none of the changes are stored. + """ + md = self.open(role, init) + with suppress(AbortEdit): + yield md.signed + self.close(role, md) + + def sign(self, role: str) -> None: + """sign without modifying content, or removing existing signatures""" + md = self.open(role) + self.close(role, md, sign_only=True) + + def snapshot( + self, current_targets: Dict[str, MetaFile] + ) -> Tuple[Optional[int], Dict[str, MetaFile]]: + """Update snapshot meta information + + Updates the meta information in snapshot according to input. + + Arguments: + current_targets: The new currently served targets roles. + + Returns: Tuple of + - New snapshot version or None if snapshot was not created + - Meta information for targets metadata that were removed from repository + """ + + # Snapshot update is needed if + # * any targets files are not yet in snapshot or + # * any targets version is incorrect + updated_snapshot = False + removed: Dict[str, MetaFile] = {} + + with self.edit("snapshot") as snapshot: + for keyname, new_meta in current_targets.items(): + if keyname not in snapshot.meta: + updated_snapshot = True + snapshot.meta[keyname] = new_meta + continue + + old_meta = snapshot.meta[keyname] + if new_meta.version < old_meta.version: + raise ValueError(f"{keyname} version rollback") + if new_meta.version > old_meta.version: + updated_snapshot = True + snapshot.meta[keyname] = new_meta + removed[keyname] = old_meta + + if not updated_snapshot: + # prevent edit() from storing a new snapshot version + raise AbortEdit("Skip snapshot: No targets version changes") + + if not updated_snapshot: + # This code is reacheable as edit() handles AbortEdit + logger.debug("Snapshot update not needed") # type: ignore[unreachable] + else: + logger.debug( + "Snapshot v%d, %d targets", snapshot.version, len(snapshot.meta) + ) + + version = snapshot.version if updated_snapshot else None + return version, removed + + def timestamp(self, snapshot_meta: MetaFile) -> Optional[MetaFile]: + """Update timestamp meta information + + Updates timestamp with given snapshot information. + + Returns the snapshot that was removed from repository (if any). + """ + with self.edit("timestamp") as timestamp: + old_snapshot_meta = timestamp.snapshot_meta + timestamp.snapshot_meta = snapshot_meta + + logger.debug("Timestamp v%d", timestamp.version) + if old_snapshot_meta.version == snapshot_meta.version: + return None + return old_snapshot_meta From 314efaf3da3f67cf75901bcba2b7a91c2dd30ee1 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Thu, 24 Nov 2022 17:06:33 +0200 Subject: [PATCH 03/15] Examples: Add repository application example This uses the repository module to create an app that * generates everything from scratch * serves metadata and targets from memory * simulates a live repository by adding new targets every few seconds Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 121 +++++++++++++++++++++++++++++ examples/repository/repo | 110 ++++++++++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 examples/repository/_simplerepo.py create mode 100755 examples/repository/repo diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py new file mode 100644 index 0000000000..225222a08a --- /dev/null +++ b/examples/repository/_simplerepo.py @@ -0,0 +1,121 @@ +# Copyright 2021-2022 python-tuf contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +"""Simple example of using the repository library to build a repository""" + +import copy +import logging +from collections import defaultdict +from datetime import datetime, timedelta +from typing import Dict, List + +from securesystemslib import keys +from securesystemslib.signer import Signer, SSlibSigner + +from tuf.api.metadata import ( + Key, + Metadata, + MetaFile, + Root, + Snapshot, + TargetFile, + Targets, + Timestamp, +) +from tuf.repository import Repository + +logger = logging.getLogger(__name__) + +_signed_init = { + Root.type: Root, + Snapshot.type: Snapshot, + Targets.type: Targets, + Timestamp.type: Timestamp, +} + + +class SimpleRepository(Repository): + """Very simple in-memory repository implementation + + This repository keeps the metadata for all versions of all roles in memory. + It also keeps all target content in memory. + + + Attributes: + role_cache: Contains every historical metadata version of every role in + this repositorys. Keys are rolenames and values are lists of + Metadata + signer_cache: Contains all signers available to the repository. Keys + are rolenames, values are lists of signers + target_cache: + """ + + expiry_period = timedelta(days=1) + + def __init__(self) -> None: + # all versions of all metadata + self.role_cache: Dict[str, List[Metadata]] = defaultdict(list) + # all current keys + self.signer_cache: Dict[str, List[Signer]] = defaultdict(list) + # all target content + self.target_cache: Dict[str, bytes] = {} + + # setup a basic repository, generate signing key per top-level role + with self.edit("root", init=True) as root: + for role in ["root", "timestamp", "snapshot", "targets"]: + key = keys.generate_ed25519_key() + self.signer_cache[role].append(SSlibSigner(key)) + root.add_key(Key.from_securesystemslib_key(key), role) + + for role in ["timestamp", "snapshot", "targets"]: + with self.edit(role, init=True): + pass + + def open(self, role: str, init: bool = False) -> Metadata: + """Return current Metadata for role from 'storage' (or create a new one)""" + + if init: + signed_init = _signed_init.get(role, Targets) + md = Metadata(signed_init()) + + # this makes version bumping in close() simpler + md.signed.version = 0 + return md + + # return latest metadata from storage (but don't return a reference) + return copy.deepcopy(self.role_cache[role][-1]) + + def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: + """Store a version of metadata. Handle version bumps, expiry, signing""" + if sign_only: + for signer in self.signer_cache[role]: + md.sign(signer, append=True) + self.role_cache[role][-1] = md + else: + md.signed.version += 1 + md.signed.expires = datetime.utcnow() + self.expiry_period + + md.signatures.clear() + for signer in self.signer_cache[role]: + md.sign(signer, append=True) + + self.role_cache[role].append(md) + + def add_target(self, path: str, content: str) -> None: + """Add a target to repository""" + data = bytes(content, "utf-8") + + # add content to cache for serving to clients + self.target_cache[path] = data + + # add a target in the targets metadata + with self.edit("targets") as targets: + targets.targets[path] = TargetFile.from_data(path, data) + + logger.debug("Targets v%d", targets.version) + + # update snapshot, timestamp + meta = {"targets.json": MetaFile(targets.version)} + new_version, _ = self.snapshot(meta) + if new_version is not None: + self.timestamp(MetaFile(new_version)) diff --git a/examples/repository/repo b/examples/repository/repo new file mode 100755 index 0000000000..1d7d53c8f1 --- /dev/null +++ b/examples/repository/repo @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# Copyright 2021-2022 python-tuf contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +"""Simple repository example application + +The application stores metadata and targets in memory, and serves them via http. +* Keys are generated at startup +* The application simulates a live reposittory by adding a new target every few seconds +""" + +import argparse +import logging +import sys +from datetime import datetime +from http.server import BaseHTTPRequestHandler, HTTPServer +from time import time +from typing import Dict, List + +from _simplerepo import SimpleRepository + +logger = logging.getLogger(__name__) + +class ReqHandler(BaseHTTPRequestHandler): + """HTTP handler to serve metadata and targets from a SimpleRepository""" + + def do_GET(self): + if self.path.startswith("/metadata/") and self.path.endswith(".json"): + self.get_metadata(self.path[len("/metadata/"):-len(".json")]) + elif self.path.startswith("/targets/"): + self.get_target(self.path[len("/targets/"):]) + else: + self.send_error(404, "Only serving /metadata/*.json") + + def get_metadata(self, ver_and_role: str): + repo = self.server.repo + + ver_str, sep, role = ver_and_role.rpartition(".") + if sep == "": + # 0 will lead to list lookup with -1, meaning latest version + ver = 0 + else: + ver = int(ver_str) + + if role not in repo.role_cache or ver > len(repo.role_cache[role]): + self.send_error(404, f"Role {role} version {ver} not found") + return + + # send the metadata json + data = repo.role_cache[role][ver-1].to_bytes() + self.send_response(200) + self.send_header('Content-length', len(data)) + self.end_headers() + self.wfile.write(data) + + def get_target(self, targetpath: str): + repo: SimpleRepository = self.server.repo + _hash, _, target = targetpath.partition(".") + + if target not in repo.target_cache: + self.send_error(404, f"target {targetpath} not found") + return + + # TODO: check that hash actually matches -- or use hash.targetpath as target_cache keys? + + # send the target content + data = repo.target_cache[target] + self.send_response(200) + self.send_header('Content-length', len(data)) + self.end_headers() + self.wfile.write(data) + + +class RepositoryServer(HTTPServer): + def __init__(self, port: int): + super().__init__(("127.0.0.1", port), ReqHandler) + self.timeout = 1 + self.repo = SimpleRepository() + + +def main(argv: List[str]) -> None: + """Example repository server""" + + parser = argparse.ArgumentParser() + parser.add_argument("-v", "--verbose", action="count") + parser.add_argument("-p", "--port", type=int, default=8001) + args, _ = parser.parse_known_args(argv) + + level = logging.DEBUG if args.verbose else logging.INFO + logging.basicConfig(level=level) + + server = RepositoryServer(args.port) + last_change = 0 + counter = 0 + + logger.info(f"Now serving. Root v1 at http://127.0.0.1:{server.server_port}/metadata/1.root.json") + + while True: + # Simulate a live repository: Add a new target file every few seconds + if time() - last_change > 10: + last_change = int(time()) + counter += 1 + content = str(datetime.fromtimestamp(last_change)) + server.repo.add_target(f"file{str(counter)}.txt", content) + + server.handle_request() + + +if __name__ == "__main__": + main(sys.argv) From 5d831537f3eb4313bd5e884e9360530789803cf0 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Thu, 24 Nov 2022 17:09:57 +0200 Subject: [PATCH 04/15] examples: Update client example * Support any repository (that serves /targets/ and /metadata/) with --url * Support multiple repositories by aking the local cache repository-specific * Add "tofu" command to initialize with Trust-On-First-Use * Update README so it uses the new repository application example Signed-off-by: Jussi Kukkonen --- examples/README.md | 1 + examples/client_example/1.root.json | 87 ------------------- examples/client_example/README.md | 42 ++++++--- .../{client_example.py => client} | 83 +++++++++++++----- 4 files changed, 91 insertions(+), 122 deletions(-) delete mode 100644 examples/client_example/1.root.json rename examples/client_example/{client_example.py => client} (56%) diff --git a/examples/README.md b/examples/README.md index fd1a1db1a1..2ad5a327bf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,5 @@ # Usage examples +* [repository](repository) * [client](client_example) * [repository built with low-level Metadata API](manual_repo) diff --git a/examples/client_example/1.root.json b/examples/client_example/1.root.json deleted file mode 100644 index 214d8db01b..0000000000 --- a/examples/client_example/1.root.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "signatures": [ - { - "keyid": "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb", - "sig": "a337d6375fedd2eabfcd6c2ef6c8a9c3bb85dc5a857715f6a6bd41123e7670c4972d8548bcd7248154f3d864bf25f1823af59d74c459f41ea09a02db057ca1245612ebbdb97e782c501dc3e094f7fa8aa1402b03c6ed0635f565e2a26f9f543a89237e15a2faf0c267e2b34c3c38f2a43a28ddcdaf8308a12ead8c6dc47d1b762de313e9ddda8cc5bc25aea1b69d0e5b9199ca02f5dda48c3bff615fd12a7136d00634b9abc6e75c3256106c4d6f12e6c43f6195071355b2857bbe377ce028619b58837696b805040ce144b393d50a472531f430fadfb68d3081b6a8b5e49337e328c9a0a3f11e80b0bc8eb2dc6e78d1451dd857e6e6e6363c3fd14c590aa95e083c9bfc77724d78af86eb7a7ef635eeddaa353030c79f66b3ba9ea11fab456cfe896a826fdfb50a43cd444f762821aada9bcd7b022c0ee85b8768f960343d5a1d3d76374cc0ac9e12a500de0bf5d48569e5398cadadadab045931c398e3bcb6cec88af2437ba91959f956079cbed159fed3938016e6c3b5e446131f81cc5981" - } - ], - "signed": { - "_type": "root", - "consistent_snapshot": false, - "expires": "2030-01-01T00:00:00Z", - "keys": { - "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "rsa", - "keyval": { - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0GjPoVrjS9eCqzoQ8VRe\nPkC0cI6ktiEgqPfHESFzyxyjC490Cuy19nuxPcJuZfN64MC48oOkR+W2mq4pM51i\nxmdG5xjvNOBRkJ5wUCc8fDCltMUTBlqt9y5eLsf/4/EoBU+zC4SW1iPU++mCsity\nfQQ7U6LOn3EYCyrkH51hZ/dvKC4o9TPYMVxNecJ3CL1q02Q145JlyjBTuM3Xdqsa\nndTHoXSRPmmzgB/1dL/c4QjMnCowrKW06mFLq9RAYGIaJWfM/0CbrOJpVDkATmEc\nMdpGJYDfW/sRQvRdlHNPo24ZW7vkQUCqdRxvnTWkK5U81y7RtjLt1yskbWXBIbOV\nz94GXsgyzANyCT9qRjHXDDz2mkLq+9I2iKtEqaEePcWRu3H6RLahpM/TxFzw684Y\nR47weXdDecPNxWyiWiyMGStRFP4Cg9trcwAGnEm1w8R2ggmWphznCd5dXGhPNjfA\na82yNFY8ubnOUVJOf0nXGg3Edw9iY3xyjJb2+nrsk5f3AgMBAAE=\n-----END PUBLIC KEY-----" - }, - "scheme": "rsassa-pss-sha256" - }, - "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ed25519", - "keyval": { - "public": "edcd0a32a07dce33f7c7873aaffbff36d20ea30787574ead335eefd337e4dacd" - }, - "scheme": "ed25519" - }, - "65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ed25519", - "keyval": { - "public": "89f28bd4ede5ec3786ab923fd154f39588d20881903e69c7b08fb504c6750815" - }, - "scheme": "ed25519" - }, - "8a1c4a3ac2d515dec982ba9910c5fd79b91ae57f625b9cff25d06bf0a61c1758": { - "keyid_hash_algorithms": [ - "sha256", - "sha512" - ], - "keytype": "ed25519", - "keyval": { - "public": "82ccf6ac47298ff43bfa0cd639868894e305a99c723ff0515ae2e9856eb5bbf4" - }, - "scheme": "ed25519" - } - }, - "roles": { - "root": { - "keyids": [ - "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb" - ], - "threshold": 1 - }, - "snapshot": { - "keyids": [ - "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d" - ], - "threshold": 1 - }, - "targets": { - "keyids": [ - "65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093" - ], - "threshold": 1 - }, - "timestamp": { - "keyids": [ - "8a1c4a3ac2d515dec982ba9910c5fd79b91ae57f625b9cff25d06bf0a61c1758" - ], - "threshold": 1 - } - }, - "spec_version": "1.0.0", - "version": 1 - } -} \ No newline at end of file diff --git a/examples/client_example/README.md b/examples/client_example/README.md index 399c6d6b42..05387a79f8 100644 --- a/examples/client_example/README.md +++ b/examples/client_example/README.md @@ -4,23 +4,43 @@ TUF Client Example, using ``python-tuf``. This TUF Client Example implements the following actions: - - Client Infrastructure Initialization - - Download target files from TUF Repository + - Client Initialization + - Target file download -The example client expects to find a TUF repository running on localhost. We -can use the static metadata files in ``tests/repository_data/repository`` -to set one up. +The client can be used against any TUF repository that serves metadata and +targets under the same URL (in _/metadata/_ and _/targets/_ directories, respectively). The +used TUF repository can be set with `--url` (default repository is "http://127.0.0.1:8001" +which is also the default for the repository example). -Run the repository using the Python3 built-in HTTP module, and keep this -session running. +### Example with the repository example + +In one terminal, run the repository example and leave it running: ```console - $ python3 -m http.server -d tests/repository_data/repository - Serving HTTP on :: port 8000 (http://[::]:8000/) ... +examples/repository/repo ``` -How to use the TUF Client Example to download a target file. +In another terminal, run the client: ```console -$ ./client_example.py download file1.txt +# initialize the client with Trust-On-First-Use +./client tofu + +# Then download example files from the repository: +./client download file1.txt +``` + +Note that unlike normal repositories, the example repository only exists in +memory and is re-generated from scratch at every startup: This means your +client needs to run `tofu` everytime you restart the repository application. + + +### Example with a repository on the internet + +```console +# On first use only, initialize the client with Trust-On-First-Use +./client --url https://jku.github.io/tuf-demo tofu + +# Then download example files from the repository: +./client --url https://jku.github.io/tuf-demo download demo/succinctly-delegated-1.txt ``` diff --git a/examples/client_example/client_example.py b/examples/client_example/client similarity index 56% rename from examples/client_example/client_example.py rename to examples/client_example/client index ffa6a989ab..faabf261fd 100755 --- a/examples/client_example/client_example.py +++ b/examples/client_example/client @@ -7,40 +7,49 @@ import argparse import logging import os -import shutil +import sys import traceback +from hashlib import sha256 from pathlib import Path +from urllib import request from tuf.api.exceptions import DownloadError, RepositoryError from tuf.ngclient import Updater # constants -BASE_URL = "http://127.0.0.1:8000" DOWNLOAD_DIR = "./downloads" -METADATA_DIR = f"{Path.home()}/.local/share/python-tuf-client-example" CLIENT_EXAMPLE_DIR = os.path.dirname(os.path.abspath(__file__)) +def build_metadata_dir(base_url: str) -> str: + """build a unique and reproducible directory name for the repository url""" + name = sha256(base_url.encode()).hexdigest()[:8] + # TODO: Make this not windows hostile? + return f"{Path.home()}/.local/share/tuf-example/{name}" -def init() -> None: - """Initialize local trusted metadata and create a directory for downloads""" + +def init_tofu(base_url: str) -> bool: + """Initialize local trusted metadata (Trust-On-First-Use) and create a + directory for downloads""" + metadata_dir = build_metadata_dir(base_url) if not os.path.isdir(DOWNLOAD_DIR): os.mkdir(DOWNLOAD_DIR) - if not os.path.isdir(METADATA_DIR): - os.makedirs(METADATA_DIR) + if not os.path.isdir(metadata_dir): + os.makedirs(metadata_dir) - if not os.path.isfile(f"{METADATA_DIR}/root.json"): - shutil.copy( - f"{CLIENT_EXAMPLE_DIR}/1.root.json", f"{METADATA_DIR}/root.json" - ) - print(f"Added trusted root in {METADATA_DIR}") + root_url = f"{base_url}/metadata/1.root.json" + try: + request.urlretrieve(root_url, f"{metadata_dir}/root.json") + except OSError: + print(f"Failed to download initial root from {root_url}") + return False - else: - print(f"Found trusted root in {METADATA_DIR}") + print(f"Trust-on-First-Use: Initialized new root in {metadata_dir}") + return True -def download(target: str) -> bool: +def download(base_url: str, target: str) -> bool: """ Download the target file using ``ngclient`` Updater. @@ -51,11 +60,23 @@ def download(target: str) -> bool: Returns: A boolean indicating if process was successful """ + metadata_dir = build_metadata_dir(base_url) + + if not os.path.isfile(f"{metadata_dir}/root.json"): + print( + "Trusted local root not found. Use 'tofu' command to " + "Trust-On-First-Use or copy trusted root metadata to " + f"{metadata_dir}/root.json" + ) + return False + + print(f"Using trusted root in {metadata_dir}") + try: updater = Updater( - metadata_dir=METADATA_DIR, - metadata_base_url=f"{BASE_URL}/metadata/", - target_base_url=f"{BASE_URL}/targets/", + metadata_dir=metadata_dir, + metadata_base_url=f"{base_url}/metadata/", + target_base_url=f"{base_url}/targets/", target_dir=DOWNLOAD_DIR, ) updater.refresh() @@ -97,9 +118,22 @@ def main() -> None: default=0, ) + client_args.add_argument( + "-u", + "--url", + help="Base repository URL", + default="http://127.0.0.1:8001", + ) + # Sub commands sub_command = client_args.add_subparsers(dest="sub_command") + # Trust-On-First-Use + sub_command.add_parser( + "tofu", + help="Initialize client with Trust-On-First-Use", + ) + # Download download_parser = sub_command.add_parser( "download", @@ -126,14 +160,15 @@ def main() -> None: logging.basicConfig(level=loglevel) # initialize the TUF Client Example infrastructure - init() - - if command_args.sub_command == "download": - download(command_args.target) - + if command_args.sub_command == "tofu": + if not init_tofu(command_args.url): + return "Failed to initialize local repository" + elif command_args.sub_command == "download": + if not download(command_args.url, command_args.target): + return f"Failed to download {command_args.target}" else: client_args.print_help() if __name__ == "__main__": - main() + sys.exit(main()) From df6b044c5a0ecf5b3699d995b58560923a56bac1 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 30 Nov 2022 11:25:31 +0200 Subject: [PATCH 05/15] repository: Make snapshot/targets info required properties This does not make the examples simpler now, but it will when there are multiple locations where snapshot/timestamp are called. * This way the snapshot/timestamp input material is an internal detail of Repository and the call sites will be simpler. * Both methods now have a "force" argument that can be used to create a new version regardless of meta info changes * but implementations are now required to implement snapshot_info and targets_infos properties that represent the current snapshot and targets versions in the repository Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 17 +++++-- tuf/repository/_repository.py | 81 ++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index 225222a08a..66e971eb90 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -71,6 +71,17 @@ def __init__(self) -> None: with self.edit(role, init=True): pass + @property + def targets_infos(self) -> Dict[str, MetaFile]: + # TODO should track changes to snapshot meta and not recreate it here + targets: Targets = self.role_cache["targets"][-1].signed + return {"targets.json": MetaFile(targets.version)} + + @property + def snapshot_info(self) -> MetaFile: + snapshot = self.role_cache["snapshot"][-1].signed + return MetaFile(snapshot.version) + def open(self, role: str, init: bool = False) -> Metadata: """Return current Metadata for role from 'storage' (or create a new one)""" @@ -115,7 +126,5 @@ def add_target(self, path: str, content: str) -> None: logger.debug("Targets v%d", targets.version) # update snapshot, timestamp - meta = {"targets.json": MetaFile(targets.version)} - new_version, _ = self.snapshot(meta) - if new_version is not None: - self.timestamp(MetaFile(new_version)) + self.snapshot() + self.timestamp() diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index 4f1d2b98df..490c605831 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -47,6 +47,24 @@ def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: with ones from all available keys.""" raise NotImplementedError + @property + @abstractmethod + def targets_infos(self) -> Dict[str, MetaFile]: + """Returns the current targets version information + + Not that there is a difference between this and the published snapshot + meta: This dictionary reflects the targets metadata that currently + exists in the repository, but the dictionary published by snapshot() + will also include metadata that no longer exists in the repository. + """ + raise NotImplementedError + + @property + @abstractmethod + def snapshot_info(self) -> MetaFile: + """Returns the information matching current snapshot metadata""" + raise NotImplementedError + @contextmanager def edit( self, role: str, init: bool = False @@ -71,31 +89,31 @@ def sign(self, role: str) -> None: md = self.open(role) self.close(role, md, sign_only=True) - def snapshot( - self, current_targets: Dict[str, MetaFile] - ) -> Tuple[Optional[int], Dict[str, MetaFile]]: + def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: """Update snapshot meta information - Updates the meta information in snapshot according to input. + Updates the snapshot meta information according to current targets + metadata state and the current current snapshot meta information. Arguments: - current_targets: The new currently served targets roles. + force: should new snapshot version be created even if meta + information would not change? Returns: Tuple of - - New snapshot version or None if snapshot was not created - - Meta information for targets metadata that were removed from repository + - True if snapshot was created, False if not + - Meta information for targets metadata was removed from snapshot """ # Snapshot update is needed if # * any targets files are not yet in snapshot or # * any targets version is incorrect - updated_snapshot = False + update_version = force removed: Dict[str, MetaFile] = {} with self.edit("snapshot") as snapshot: - for keyname, new_meta in current_targets.items(): + for keyname, new_meta in self.targets_infos.items(): if keyname not in snapshot.meta: - updated_snapshot = True + update_version = True snapshot.meta[keyname] = new_meta continue @@ -103,37 +121,48 @@ def snapshot( if new_meta.version < old_meta.version: raise ValueError(f"{keyname} version rollback") if new_meta.version > old_meta.version: - updated_snapshot = True + update_version = True snapshot.meta[keyname] = new_meta removed[keyname] = old_meta - if not updated_snapshot: + if not update_version: # prevent edit() from storing a new snapshot version raise AbortEdit("Skip snapshot: No targets version changes") - if not updated_snapshot: - # This code is reacheable as edit() handles AbortEdit - logger.debug("Snapshot update not needed") # type: ignore[unreachable] + if not update_version: + logger.debug("Snapshot update not needed") else: logger.debug( "Snapshot v%d, %d targets", snapshot.version, len(snapshot.meta) ) - version = snapshot.version if updated_snapshot else None - return version, removed + return update_version, removed - def timestamp(self, snapshot_meta: MetaFile) -> Optional[MetaFile]: + def timestamp(self, force: bool = False) -> Tuple[bool, Optional[MetaFile]]: """Update timestamp meta information - Updates timestamp with given snapshot information. + Updates timestamp according to current snapshot state - Returns the snapshot that was removed from repository (if any). + Returns: Tuple of + - True if timestamp was created, False if not + - Meta information for snapshot metadata that was removed from timestamp """ + update_version = force + removed = None with self.edit("timestamp") as timestamp: - old_snapshot_meta = timestamp.snapshot_meta - timestamp.snapshot_meta = snapshot_meta + if self.snapshot_info.version < timestamp.snapshot_meta.version: + raise ValueError(f"snapshot version rollback") - logger.debug("Timestamp v%d", timestamp.version) - if old_snapshot_meta.version == snapshot_meta.version: - return None - return old_snapshot_meta + if self.snapshot_info.version > timestamp.snapshot_meta.version: + update_version = True + removed = timestamp.snapshot_meta + timestamp.snapshot_meta = self.snapshot_info + + if not update_version: + raise AbortEdit("Skip timestamp: No snapshot version changes") + + if not update_version: + logger.debug("Timestamp update not needed") + else: + logger.debug("Timestamp v%d", timestamp.version) + return update_version, removed From dd36b73ca936c7f05b26c03ea03889c7eda560b7 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 30 Nov 2022 21:05:57 +0200 Subject: [PATCH 06/15] repository: insert copies of MetaFile into metadata Otherwise the metafile cache and the metadata object end up pointing to same instances which starts breaking later. Signed-off-by: Jussi Kukkonen --- tuf/repository/_repository.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index 490c605831..e3418e1892 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -3,6 +3,7 @@ """Repository Abstraction for metadata management""" +from copy import deepcopy import logging from abc import ABC, abstractmethod from contextlib import contextmanager, suppress @@ -114,7 +115,7 @@ def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: for keyname, new_meta in self.targets_infos.items(): if keyname not in snapshot.meta: update_version = True - snapshot.meta[keyname] = new_meta + snapshot.meta[keyname] = deepcopy(new_meta) continue old_meta = snapshot.meta[keyname] @@ -122,7 +123,7 @@ def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: raise ValueError(f"{keyname} version rollback") if new_meta.version > old_meta.version: update_version = True - snapshot.meta[keyname] = new_meta + snapshot.meta[keyname] = deepcopy(new_meta) removed[keyname] = old_meta if not update_version: @@ -156,7 +157,7 @@ def timestamp(self, force: bool = False) -> Tuple[bool, Optional[MetaFile]]: if self.snapshot_info.version > timestamp.snapshot_meta.version: update_version = True removed = timestamp.snapshot_meta - timestamp.snapshot_meta = self.snapshot_info + timestamp.snapshot_meta = deepcopy(self.snapshot_info) if not update_version: raise AbortEdit("Skip timestamp: No snapshot version changes") From 87c74a83bc6b6238d74e05ade1724ee4f24a82ba Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 30 Nov 2022 21:07:39 +0200 Subject: [PATCH 07/15] examples: Maintain a meta info cache This is not required for the demo but is more realistic: we keep a cache of targets versions so that we can produce a new snapshot whenever one is needed, without accessing all of the targets metadata to do so. Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 15 ++++++++++----- tuf/repository/_repository.py | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index 66e971eb90..e5e0ea14f6 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -59,6 +59,9 @@ def __init__(self) -> None: self.signer_cache: Dict[str, List[Signer]] = defaultdict(list) # all target content self.target_cache: Dict[str, bytes] = {} + # version cache for snapshot and all targets, updated in close() + self._snapshot_info = MetaFile(1) + self._targets_infos = defaultdict(lambda: MetaFile(1)) # setup a basic repository, generate signing key per top-level role with self.edit("root", init=True) as root: @@ -73,14 +76,11 @@ def __init__(self) -> None: @property def targets_infos(self) -> Dict[str, MetaFile]: - # TODO should track changes to snapshot meta and not recreate it here - targets: Targets = self.role_cache["targets"][-1].signed - return {"targets.json": MetaFile(targets.version)} + return self._targets_infos @property def snapshot_info(self) -> MetaFile: - snapshot = self.role_cache["snapshot"][-1].signed - return MetaFile(snapshot.version) + return self._snapshot_info def open(self, role: str, init: bool = False) -> Metadata: """Return current Metadata for role from 'storage' (or create a new one)""" @@ -110,7 +110,12 @@ def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: for signer in self.signer_cache[role]: md.sign(signer, append=True) + # store new metadata version, update version caches self.role_cache[role].append(md) + if role == "snapshot": + self._snapshot_info.version = md.signed.version + elif role not in ["root", "timestamp"]: + self._targets_infos[f"{role}.json"].version = md.signed.version def add_target(self, path: str, content: str) -> None: """Add a target to repository""" diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index e3418e1892..aed7be283c 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -45,7 +45,8 @@ def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: If sign_only, then just append signatures of all available keys. If not sign_only, update expiry and version and replace signatures - with ones from all available keys.""" + with ones from all available keys. Keep snapshot_info and targets_infos + updated.""" raise NotImplementedError @property From 69cb140cb3c550c6500938439088886750d4f8b4 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Fri, 2 Dec 2022 13:30:05 +0200 Subject: [PATCH 08/15] examples: Add README for repository example Tweak comments as well Signed-off-by: Jussi Kukkonen --- examples/repository/README.md | 23 +++++++++++++++++++++++ examples/repository/repo | 12 ++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 examples/repository/README.md diff --git a/examples/repository/README.md b/examples/repository/README.md new file mode 100644 index 0000000000..53c292a883 --- /dev/null +++ b/examples/repository/README.md @@ -0,0 +1,23 @@ +# TUF Repository Application Example + + +This TUF Repository Application Example has following features: +- Initializes a completely new repository on startup +- Stores everything (metadata, targets, signing keys) in-memory +- Serves metadata and targets on localhost (default port 8001) +- Simulates a live repository by automatically adding a new target + file every 10 seconds. + + +### Example with the repository example + +```console +./repo +``` +Your repository is now running and is accessible on localhost, See e.g. +http://127.0.0.1:8001/metadata/1.root.json + +Note that because the example generates a new repository at startup, +clients need to also re-initialize their trust root when the repository +application is restarted. With the example client this is done with +`./client tofu`. diff --git a/examples/repository/repo b/examples/repository/repo index 1d7d53c8f1..50678379e0 100755 --- a/examples/repository/repo +++ b/examples/repository/repo @@ -5,8 +5,8 @@ """Simple repository example application The application stores metadata and targets in memory, and serves them via http. -* Keys are generated at startup -* The application simulates a live reposittory by adding a new target every few seconds +Nothing is persisted on disk or loaded from disk. The application simulates a +live repository by adding new target files periodically. """ import argparse @@ -28,7 +28,7 @@ class ReqHandler(BaseHTTPRequestHandler): if self.path.startswith("/metadata/") and self.path.endswith(".json"): self.get_metadata(self.path[len("/metadata/"):-len(".json")]) elif self.path.startswith("/targets/"): - self.get_target(self.path[len("/targets/"):]) + self.get_target(self.path[len("/targets/"):]) else: self.send_error(404, "Only serving /metadata/*.json") @@ -46,7 +46,7 @@ class ReqHandler(BaseHTTPRequestHandler): self.send_error(404, f"Role {role} version {ver} not found") return - # send the metadata json + # send the metadata json data = repo.role_cache[role][ver-1].to_bytes() self.send_response(200) self.send_header('Content-length', len(data)) @@ -80,7 +80,7 @@ class RepositoryServer(HTTPServer): def main(argv: List[str]) -> None: """Example repository server""" - + parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="count") parser.add_argument("-p", "--port", type=int, default=8001) @@ -92,7 +92,7 @@ def main(argv: List[str]) -> None: server = RepositoryServer(args.port) last_change = 0 counter = 0 - + logger.info(f"Now serving. Root v1 at http://127.0.0.1:{server.server_port}/metadata/1.root.json") while True: From 0f94c03756f97d0fc0c114753f3a8f2bc34d1f0a Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Fri, 2 Dec 2022 13:40:58 +0200 Subject: [PATCH 09/15] repository: Handle linting issues Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 4 +++- tuf/repository/_repository.py | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index e5e0ea14f6..7c1e54f1c1 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -61,7 +61,9 @@ def __init__(self) -> None: self.target_cache: Dict[str, bytes] = {} # version cache for snapshot and all targets, updated in close() self._snapshot_info = MetaFile(1) - self._targets_infos = defaultdict(lambda: MetaFile(1)) + self._targets_infos: Dict[str, MetaFile] = defaultdict( + lambda: MetaFile(1) + ) # setup a basic repository, generate signing key per top-level role with self.edit("root", init=True) as root: diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index aed7be283c..9d32a82b4f 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -3,10 +3,10 @@ """Repository Abstraction for metadata management""" -from copy import deepcopy import logging from abc import ABC, abstractmethod from contextlib import contextmanager, suppress +from copy import deepcopy from typing import Dict, Generator, Optional, Tuple from tuf.api.metadata import Metadata, MetaFile, Signed @@ -132,7 +132,8 @@ def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: raise AbortEdit("Skip snapshot: No targets version changes") if not update_version: - logger.debug("Snapshot update not needed") + # this is reachable as edit() handles AbortEdit + logger.debug("Snapshot update not needed") # type: ignore[unreachable] else: logger.debug( "Snapshot v%d, %d targets", snapshot.version, len(snapshot.meta) @@ -153,7 +154,7 @@ def timestamp(self, force: bool = False) -> Tuple[bool, Optional[MetaFile]]: removed = None with self.edit("timestamp") as timestamp: if self.snapshot_info.version < timestamp.snapshot_meta.version: - raise ValueError(f"snapshot version rollback") + raise ValueError("snapshot version rollback") if self.snapshot_info.version > timestamp.snapshot_meta.version: update_version = True @@ -164,7 +165,8 @@ def timestamp(self, force: bool = False) -> Tuple[bool, Optional[MetaFile]]: raise AbortEdit("Skip timestamp: No snapshot version changes") if not update_version: - logger.debug("Timestamp update not needed") + # this is reachable as edit() handles AbortEdit + logger.debug("Timestamp update not needed") # type: ignore[unreachable] else: logger.debug("Timestamp v%d", timestamp.version) return update_version, removed From fdf0affcad8a3761dc3e60ddb4a6bf5ce154cc8b Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Sat, 3 Dec 2022 11:33:06 +0200 Subject: [PATCH 10/15] repository: Address review comments This is a collection of comment, documentation and logging fixes. The noteworthy part is making it clear that repository is not stable API yet: I think this is a good idea. Signed-off-by: Jussi Kukkonen --- examples/client_example/README.md | 2 +- examples/repository/README.md | 14 ++++++-------- examples/repository/_simplerepo.py | 12 ++++++------ tuf/repository/__init__.py | 9 ++++++++- tuf/repository/_repository.py | 11 ++++++----- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/examples/client_example/README.md b/examples/client_example/README.md index 05387a79f8..6674984de9 100644 --- a/examples/client_example/README.md +++ b/examples/client_example/README.md @@ -32,7 +32,7 @@ In another terminal, run the client: Note that unlike normal repositories, the example repository only exists in memory and is re-generated from scratch at every startup: This means your -client needs to run `tofu` everytime you restart the repository application. +client needs to run `tofu` every time you restart the repository application. ### Example with a repository on the internet diff --git a/examples/repository/README.md b/examples/repository/README.md index 53c292a883..9b9b92626f 100644 --- a/examples/repository/README.md +++ b/examples/repository/README.md @@ -1,7 +1,9 @@ # TUF Repository Application Example +:warning: This example uses the repository module which is not considered +part of the python-tuf stable API quite yet. -This TUF Repository Application Example has following features: +This TUF Repository Application Example has the following features: - Initializes a completely new repository on startup - Stores everything (metadata, targets, signing keys) in-memory - Serves metadata and targets on localhost (default port 8001) @@ -9,15 +11,11 @@ This TUF Repository Application Example has following features: file every 10 seconds. -### Example with the repository example +### Usage ```console ./repo ``` Your repository is now running and is accessible on localhost, See e.g. -http://127.0.0.1:8001/metadata/1.root.json - -Note that because the example generates a new repository at startup, -clients need to also re-initialize their trust root when the repository -application is restarted. With the example client this is done with -`./client tofu`. +http://127.0.0.1:8001/metadata/1.root.json. The +[client example](../client_example/README.md) uses this address by default. diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index 7c1e54f1c1..57dacdf4bc 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -42,12 +42,12 @@ class SimpleRepository(Repository): Attributes: - role_cache: Contains every historical metadata version of every role in - this repositorys. Keys are rolenames and values are lists of - Metadata - signer_cache: Contains all signers available to the repository. Keys - are rolenames, values are lists of signers - target_cache: + role_cache: Every historical metadata version of every role in this + repositorys. Keys are role names and values are lists of Metadata + signer_cache: All signers available to the repository. Keys are role + names, values are lists of signers + target_cache: All target files served by the repository. Keys are + target paths and values are file contents as bytes. """ expiry_period = timedelta(days=1) diff --git a/tuf/repository/__init__.py b/tuf/repository/__init__.py index ee28015fce..57b29f1108 100644 --- a/tuf/repository/__init__.py +++ b/tuf/repository/__init__.py @@ -1,6 +1,13 @@ # Copyright 2021-2022 python-tuf contributors # SPDX-License-Identifier: MIT OR Apache-2.0 -"""Repository API: A library to help repository implementations""" +"""Repository API: A helper library for repository implementations + +This module is intended to make any "metadata editing" applications easier to +implement: this includes repository applications, CI integration components as +well as developer and signing tools. + +The repository module is not considered part of the stable python-tuf API yet. +""" from tuf.repository._repository import AbortEdit, Repository diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index 9d32a82b4f..35a4762a8d 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -21,6 +21,9 @@ class AbortEdit(Exception): class Repository(ABC): """Abstract class for metadata modifying implementations + NOTE: The repository module is not considered part of the python-tuf + stable API yet. + This class is intended to be a base class used in any metadata editing application, whether it is a real repository server or a developer tool. @@ -95,7 +98,7 @@ def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: """Update snapshot meta information Updates the snapshot meta information according to current targets - metadata state and the current current snapshot meta information. + metadata state and the current snapshot meta information. Arguments: force: should new snapshot version be created even if meta @@ -103,7 +106,7 @@ def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: Returns: Tuple of - True if snapshot was created, False if not - - Meta information for targets metadata was removed from snapshot + - Meta information for targets metadata that was removed from snapshot """ # Snapshot update is needed if @@ -135,9 +138,7 @@ def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: # this is reachable as edit() handles AbortEdit logger.debug("Snapshot update not needed") # type: ignore[unreachable] else: - logger.debug( - "Snapshot v%d, %d targets", snapshot.version, len(snapshot.meta) - ) + logger.debug("Snapshot v%d", snapshot.version) return update_version, removed From 3e4ef61e4656077a0a3f0ce8325d4a9a3e35ca99 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Mon, 5 Dec 2022 12:19:33 +0200 Subject: [PATCH 11/15] examples: Tweak client README Signed-off-by: Jussi Kukkonen --- examples/client_example/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/client_example/README.md b/examples/client_example/README.md index 6674984de9..3a7ba603a6 100644 --- a/examples/client_example/README.md +++ b/examples/client_example/README.md @@ -13,7 +13,7 @@ used TUF repository can be set with `--url` (default repository is "http://127.0 which is also the default for the repository example). -### Example with the repository example +### Usage with the repository example In one terminal, run the repository example and leave it running: ```console @@ -35,7 +35,7 @@ memory and is re-generated from scratch at every startup: This means your client needs to run `tofu` every time you restart the repository application. -### Example with a repository on the internet +### Usage with a repository on the internet ```console # On first use only, initialize the client with Trust-On-First-Use From c1bb46b6c2514f90a343a5552a4abb2f6ed3f19e Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Mon, 5 Dec 2022 13:43:15 +0200 Subject: [PATCH 12/15] repository: Improve docstrings Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 4 +++- tuf/repository/_repository.py | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index 57dacdf4bc..916905bda7 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -59,7 +59,9 @@ def __init__(self) -> None: self.signer_cache: Dict[str, List[Signer]] = defaultdict(list) # all target content self.target_cache: Dict[str, bytes] = {} - # version cache for snapshot and all targets, updated in close() + # version cache for snapshot and all targets, updated in close(). + # The 'defaultdict(lambda: ...)' trick allows close() to easily modify + # the version without always creating a new MetaFile self._snapshot_info = MetaFile(1) self._targets_infos: Dict[str, MetaFile] = defaultdict( lambda: MetaFile(1) diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index 35a4762a8d..299c7cbf6a 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -55,26 +55,31 @@ def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: @property @abstractmethod def targets_infos(self) -> Dict[str, MetaFile]: - """Returns the current targets version information + """Returns the MetaFiles for current targets metadatas - Not that there is a difference between this and the published snapshot - meta: This dictionary reflects the targets metadata that currently - exists in the repository, but the dictionary published by snapshot() - will also include metadata that no longer exists in the repository. + This property is used by snapshot() to update Snapshot.meta. + + Note that there is a difference between this return value and + Snapshot.meta: This dictionary reflects the targets metadata that + currently exists in the repository but Snapshot.meta also includes + metadata that used to exist, but no longer exists, in the repository. """ raise NotImplementedError @property @abstractmethod def snapshot_info(self) -> MetaFile: - """Returns the information matching current snapshot metadata""" + """Returns the MetaFile for current snapshot metadata + + This property is used by timestamp() to update Timestamp.meta. + """ raise NotImplementedError @contextmanager def edit( self, role: str, init: bool = False ) -> Generator[Signed, None, None]: - """Context manager for editing a roles metadata + """Context manager for editing a role's metadata Context manager takes care of loading the roles metadata (or creating new metadata if 'init'), updating expiry and version. The caller can do @@ -106,7 +111,7 @@ def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: Returns: Tuple of - True if snapshot was created, False if not - - Meta information for targets metadata that was removed from snapshot + - MetaFiles for targets versions removed from snapshot meta """ # Snapshot update is needed if @@ -149,7 +154,7 @@ def timestamp(self, force: bool = False) -> Tuple[bool, Optional[MetaFile]]: Returns: Tuple of - True if timestamp was created, False if not - - Meta information for snapshot metadata that was removed from timestamp + - MetaFile for snapshot version removed from timestamp (if any) """ update_version = force removed = None From 9e9c1562882b36cb4be5976be25d5c2e5c1a1da3 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 14 Dec 2022 19:53:43 +0200 Subject: [PATCH 13/15] repository: remove init argument from open() This no longer seems needed: if the metadata store does not contain a single version of role, then open() can assume it is initializing. Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 8 ++++---- examples/repository/repo | 15 +++++++++------ tuf/repository/_repository.py | 12 +++++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index 916905bda7..b81d740e04 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -68,14 +68,14 @@ def __init__(self) -> None: ) # setup a basic repository, generate signing key per top-level role - with self.edit("root", init=True) as root: + with self.edit("root") as root: for role in ["root", "timestamp", "snapshot", "targets"]: key = keys.generate_ed25519_key() self.signer_cache[role].append(SSlibSigner(key)) root.add_key(Key.from_securesystemslib_key(key), role) for role in ["timestamp", "snapshot", "targets"]: - with self.edit(role, init=True): + with self.edit(role): pass @property @@ -86,10 +86,10 @@ def targets_infos(self) -> Dict[str, MetaFile]: def snapshot_info(self) -> MetaFile: return self._snapshot_info - def open(self, role: str, init: bool = False) -> Metadata: + def open(self, role: str) -> Metadata: """Return current Metadata for role from 'storage' (or create a new one)""" - if init: + if role not in self.role_cache: signed_init = _signed_init.get(role, Targets) md = Metadata(signed_init()) diff --git a/examples/repository/repo b/examples/repository/repo index 50678379e0..361151233a 100755 --- a/examples/repository/repo +++ b/examples/repository/repo @@ -21,14 +21,15 @@ from _simplerepo import SimpleRepository logger = logging.getLogger(__name__) + class ReqHandler(BaseHTTPRequestHandler): """HTTP handler to serve metadata and targets from a SimpleRepository""" def do_GET(self): if self.path.startswith("/metadata/") and self.path.endswith(".json"): - self.get_metadata(self.path[len("/metadata/"):-len(".json")]) + self.get_metadata(self.path[len("/metadata/") : -len(".json")]) elif self.path.startswith("/targets/"): - self.get_target(self.path[len("/targets/"):]) + self.get_target(self.path[len("/targets/") :]) else: self.send_error(404, "Only serving /metadata/*.json") @@ -47,9 +48,9 @@ class ReqHandler(BaseHTTPRequestHandler): return # send the metadata json - data = repo.role_cache[role][ver-1].to_bytes() + data = repo.role_cache[role][ver - 1].to_bytes() self.send_response(200) - self.send_header('Content-length', len(data)) + self.send_header("Content-length", len(data)) self.end_headers() self.wfile.write(data) @@ -66,7 +67,7 @@ class ReqHandler(BaseHTTPRequestHandler): # send the target content data = repo.target_cache[target] self.send_response(200) - self.send_header('Content-length', len(data)) + self.send_header("Content-length", len(data)) self.end_headers() self.wfile.write(data) @@ -93,7 +94,9 @@ def main(argv: List[str]) -> None: last_change = 0 counter = 0 - logger.info(f"Now serving. Root v1 at http://127.0.0.1:{server.server_port}/metadata/1.root.json") + logger.info( + f"Now serving. Root v1 at http://127.0.0.1:{server.server_port}/metadata/1.root.json" + ) while True: # Simulate a live repository: Add a new target file every few seconds diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index 299c7cbf6a..f1fa6fc823 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -35,10 +35,10 @@ class Repository(ABC): """ @abstractmethod - def open(self, role: str, init: bool = False) -> Metadata: + def open(self, role: str) -> Metadata: """Load a roles metadata from storage or cache, return it - If 'init', then create metadata from scratch""" + If role has no metadata, create first version from scratch""" raise NotImplementedError @abstractmethod @@ -76,20 +76,18 @@ def snapshot_info(self) -> MetaFile: raise NotImplementedError @contextmanager - def edit( - self, role: str, init: bool = False - ) -> Generator[Signed, None, None]: + def edit(self, role: str) -> Generator[Signed, None, None]: """Context manager for editing a role's metadata Context manager takes care of loading the roles metadata (or creating - new metadata if 'init'), updating expiry and version. The caller can do + new metadata), updating expiry and version. The caller can do other changes to the Signed object and when the context manager exits, a new version of the roles metadata is stored. Context manager user can raise AbortEdit from inside the with-block to cancel the edit: in this case none of the changes are stored. """ - md = self.open(role, init) + md = self.open(role) with suppress(AbortEdit): yield md.signed self.close(role, md) From 48865aede9336687d83911d76c6f8fbf31086563 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 14 Dec 2022 20:05:56 +0200 Subject: [PATCH 14/15] repository: Remove sign_only argument from close() This is only needed for threshold signing and not even used in the example: leave it to the implementations to handle for now. Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 33 +++++++++++++----------------- tuf/repository/_repository.py | 13 +++++------- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index b81d740e04..c78d33c67e 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -100,26 +100,21 @@ def open(self, role: str) -> Metadata: # return latest metadata from storage (but don't return a reference) return copy.deepcopy(self.role_cache[role][-1]) - def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: + def close(self, role: str, md: Metadata) -> None: """Store a version of metadata. Handle version bumps, expiry, signing""" - if sign_only: - for signer in self.signer_cache[role]: - md.sign(signer, append=True) - self.role_cache[role][-1] = md - else: - md.signed.version += 1 - md.signed.expires = datetime.utcnow() + self.expiry_period - - md.signatures.clear() - for signer in self.signer_cache[role]: - md.sign(signer, append=True) - - # store new metadata version, update version caches - self.role_cache[role].append(md) - if role == "snapshot": - self._snapshot_info.version = md.signed.version - elif role not in ["root", "timestamp"]: - self._targets_infos[f"{role}.json"].version = md.signed.version + md.signed.version += 1 + md.signed.expires = datetime.utcnow() + self.expiry_period + + md.signatures.clear() + for signer in self.signer_cache[role]: + md.sign(signer, append=True) + + # store new metadata version, update version caches + self.role_cache[role].append(md) + if role == "snapshot": + self._snapshot_info.version = md.signed.version + elif role not in ["root", "timestamp"]: + self._targets_infos[f"{role}.json"].version = md.signed.version def add_target(self, path: str, content: str) -> None: """Add a target to repository""" diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index f1fa6fc823..107d92308f 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -28,9 +28,11 @@ class Repository(ABC): application, whether it is a real repository server or a developer tool. Implementations must implement open() and close(), and can then use the - edit() contextmanager to implement actual operations. + edit() contextmanager to implement actual operations. Not that signing + an already existing version of metadata (as could be done for threshold + signing) does not fit into this model of open()+close() or edit(). - A few operations (sign, snapshot and timestamp) are already implemented + A few operations (snapshot and timestamp) are already implemented in this base class. """ @@ -42,7 +44,7 @@ def open(self, role: str) -> Metadata: raise NotImplementedError @abstractmethod - def close(self, role: str, md: Metadata, sign_only: bool = False) -> None: + def close(self, role: str, md: Metadata) -> None: """Write roles metadata into storage If sign_only, then just append signatures of all available keys. @@ -92,11 +94,6 @@ def edit(self, role: str) -> Generator[Signed, None, None]: yield md.signed self.close(role, md) - def sign(self, role: str) -> None: - """sign without modifying content, or removing existing signatures""" - md = self.open(role) - self.close(role, md, sign_only=True) - def snapshot(self, force: bool = False) -> Tuple[bool, Dict[str, MetaFile]]: """Update snapshot meta information From fd02226acbda7251259c8f655cbe779e27a4a521 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Sat, 17 Dec 2022 23:09:11 +0200 Subject: [PATCH 15/15] repository: Improve dosctrings Signed-off-by: Jussi Kukkonen --- examples/repository/_simplerepo.py | 2 +- tuf/repository/_repository.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/repository/_simplerepo.py b/examples/repository/_simplerepo.py index c78d33c67e..34308ca770 100644 --- a/examples/repository/_simplerepo.py +++ b/examples/repository/_simplerepo.py @@ -43,7 +43,7 @@ class SimpleRepository(Repository): Attributes: role_cache: Every historical metadata version of every role in this - repositorys. Keys are role names and values are lists of Metadata + repository. Keys are role names and values are lists of Metadata signer_cache: All signers available to the repository. Keys are role names, values are lists of signers target_cache: All target files served by the repository. Keys are diff --git a/tuf/repository/_repository.py b/tuf/repository/_repository.py index 107d92308f..ec1223e302 100644 --- a/tuf/repository/_repository.py +++ b/tuf/repository/_repository.py @@ -28,7 +28,7 @@ class Repository(ABC): application, whether it is a real repository server or a developer tool. Implementations must implement open() and close(), and can then use the - edit() contextmanager to implement actual operations. Not that signing + edit() contextmanager to implement actual operations. Note that signing an already existing version of metadata (as could be done for threshold signing) does not fit into this model of open()+close() or edit(). @@ -47,11 +47,8 @@ def open(self, role: str) -> Metadata: def close(self, role: str, md: Metadata) -> None: """Write roles metadata into storage - If sign_only, then just append signatures of all available keys. - - If not sign_only, update expiry and version and replace signatures - with ones from all available keys. Keep snapshot_info and targets_infos - updated.""" + Update expiry and version and replace signatures with ones from all + available keys. Keep snapshot_info and targets_infos updated.""" raise NotImplementedError @property