diff --git a/Makefile b/Makefile index d80b785d2a1b..2d146038482b 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}") DB := example IPYTHON := no LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))") +WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython # needed in development environment @@ -153,6 +154,14 @@ initdb: docker-compose run --rm web python -m warehouse db upgrade head $(MAKE) reindex +inittuf: + $(WAREHOUSE_CLI) tuf keypair --rolename root + $(WAREHOUSE_CLI) tuf keypair --rolename snapshot + $(WAREHOUSE_CLI) tuf keypair --rolename targets + $(WAREHOUSE_CLI) tuf keypair --rolename timestamp + $(WAREHOUSE_CLI) tuf keypair --rolename bins + $(WAREHOUSE_CLI) tuf keypair --rolename bin-n + reindex: docker-compose run --rm web python -m warehouse search reindex diff --git a/requirements/main.in b/requirements/main.in index c95bf79c6cf2..2857f3212e2d 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -18,6 +18,7 @@ google-cloud-bigquery google-cloud-storage hiredis html5lib +hvac>=0.10.6 itsdangerous Jinja2>=2.8 limits diff --git a/requirements/main.txt b/requirements/main.txt index 46fd188e7d3c..bf03b95398c5 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -374,6 +374,10 @@ hupper==1.10.2 \ --hash=sha256:3818f53dabc24da66f65cf4878c1c7a9b5df0c46b813e014abdd7c569eb9a02a \ --hash=sha256:5de835f3b58324af2a8a16f52270c4d1a3d1734c45eed94b77fd622aea737f29 \ # via pyramid +hvac==0.10.6 \ + --hash=sha256:6e4bea65235bc38b85162a141194d07c857c6722ecd3edb6aeda316e8f4950f5 \ + --hash=sha256:b0561dbdfecc6a6d7b0cc226d75a800ae9bbc93313a6ad526a1adc97be51eada \ + # via -r requirements/main.in idna==2.10 \ --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \ @@ -727,7 +731,7 @@ requests-aws4auth==1.0.1 \ requests==2.25.0 \ --hash=sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8 \ --hash=sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998 \ - # via -r requirements/main.in, datadog, google-api-core, google-cloud-storage, premailer, requests-aws4auth + # via -r requirements/main.in, datadog, google-api-core, google-cloud-storage, hvac, premailer, requests-aws4auth rfc3986==1.4.0 \ --hash=sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d \ --hash=sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50 \ @@ -747,7 +751,7 @@ sentry-sdk==0.19.4 \ six==1.15.0 \ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ - # via argon2-cffi, automat, bcrypt, bleach, cryptography, elasticsearch-dsl, google-api-core, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, html5lib, limits, packaging, protobuf, pymacaroons, pynacl, pyopenssl, python-dateutil, readme-renderer, structlog, tenacity, webauthn + # via argon2-cffi, automat, bcrypt, bleach, cryptography, elasticsearch-dsl, google-api-core, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, html5lib, hvac, limits, packaging, protobuf, pymacaroons, pynacl, pyopenssl, python-dateutil, readme-renderer, structlog, tenacity, webauthn sqlalchemy-citext==1.7.0 \ --hash=sha256:69ba00f5505f92a1455a94eefc6d3fcf72dda3691ab5398a0b4d0d8d85bd6aab \ # via -r requirements/main.in diff --git a/tests/unit/cli/test_tuf.py b/tests/unit/cli/test_tuf.py new file mode 100644 index 000000000000..0680ad91c730 --- /dev/null +++ b/tests/unit/cli/test_tuf.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from warehouse.cli import tuf + + +class TestTufCLI: + def test_keypair(self, monkeypatch, cli): + response = pretend.stub(raise_for_status=pretend.call_recorder(lambda: None)) + client = pretend.stub( + secrets=pretend.stub( + transit=pretend.stub( + create_key=pretend.call_recorder(lambda **kw: response), + read_key=pretend.call_recorder(lambda **kw: "fake key info"), + ) + ) + ) + vault = pretend.call_recorder(lambda c: client) + monkeypatch.setattr(tuf, "_vault", vault) + + config = pretend.stub() + + result = cli.invoke(tuf.keypair, ["--rolename", "root"], obj=config) + + assert result.exit_code == 0 + assert vault.calls == [pretend.call(config)] + assert client.secrets.transit.create_key.calls == [ + pretend.call(name="root", exportable=False, key_type="ed25519") + ] + assert client.secrets.transit.read_key.calls == [pretend.call(name="root")] + assert response.raise_for_status.calls == [pretend.call()] diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index b3c2033c749a..7fc086d71c87 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -231,6 +231,8 @@ def __init__(self): "token.default.max_age": 21600, "warehouse.xmlrpc.client.ratelimit_string": "3600 per hour", "warehouse.xmlrpc.search.enabled": True, + "vault.verify": environment == config.Environment.production, + "vault.cert": None, } if environment == config.Environment.development: diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py new file mode 100644 index 000000000000..56e0bf613c8a --- /dev/null +++ b/warehouse/cli/tuf.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import hvac + +from warehouse.cli import warehouse + + +# pragma: no branch +def _vault(config): + return hvac.Client( + url=config.registry.settings["vault.url"], + token=config.registry.settings["vault.token"], + cert=config.registry.settings["vault.cert"], + verify=config.registry.settings["vault.verify"], + ) + + +@warehouse.group() +def tuf(): + """ + Manage Warehouse's TUF state. + """ + + +@tuf.command() +@click.pass_obj +@click.option( + "--rolename", required=True, help="The name of the TUF role for this keypair" +) +def keypair(config, rolename): + """ + Generate a new TUF keypair. + """ + vault = _vault(config) + resp = vault.secrets.transit.create_key( + name=rolename, exportable=False, key_type="ed25519" + ) + resp.raise_for_status() + info = vault.secrets.transit.read_key(name=rolename) + print(info) diff --git a/warehouse/config.py b/warehouse/config.py index 5dd7b4e71704..d91ff4b63dd9 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -214,6 +214,10 @@ def configure(settings=None): coercer=int, default=21600, # 6 hours ) + maybe_set(settings, "vault.url", "VAULT_URL") + maybe_set(settings, "vault.token", "VAULT_TOKEN") + maybe_set(settings, "vault.verify", "VAULT_VERIFY") + maybe_set(settings, "vault.cert", "VAULT_VERIFY") maybe_set_compound(settings, "files", "backend", "FILES_BACKEND") maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND") maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE") @@ -222,6 +226,13 @@ def configure(settings=None): maybe_set_compound(settings, "breached_passwords", "backend", "BREACHED_PASSWORDS") maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND") + # Require an encrypted connection to Vault in production. + settings.setdefault( + "vault.verify", settings["warehouse.env"] == Environment.production + ) + settings.setdefault("vault.cert", None) + maybe_set(settings, "vault.vert", "VAULT_CERT") + # Add the settings we use when the environment is set to development. if settings["warehouse.env"] == Environment.development: settings.setdefault("enforce_https", False) diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py new file mode 100644 index 000000000000..164f68b09175 --- /dev/null +++ b/warehouse/tuf/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/warehouse/tuf/interfaces.py b/warehouse/tuf/interfaces.py new file mode 100644 index 000000000000..a65d4cf8dc43 --- /dev/null +++ b/warehouse/tuf/interfaces.py @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zope.interface import Interface + + +class IKeyService(Interface): + def create_service(context, request): + """ + Create the service, given the context and request for which it is being + created. + """ + + def keys_for_role(rolename): + """ + Returns a list of TUF `api.key.Key` for the given rolename. + """ diff --git a/warehouse/tuf/services.py b/warehouse/tuf/services.py new file mode 100644 index 000000000000..3b8c1c9052c7 --- /dev/null +++ b/warehouse/tuf/services.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hvac + +from zope.interface import implementer + +from warehouse.tuf.interfaces import IKeyService + + +@implementer(IKeyService) +class VaultKeyService: + def __init__(self, request): + self._vault = hvac.Client( + url=request.registry.settings["vault.url"], + token=request.registry.settings["vault.token"], + cert=request.registry.settings["vault.cert"], + verify=request.registry.settings["vault.verify"], + ) + + @classmethod + def create_service(cls, _context, request): + return cls(request) + + def keys_for_role(self, rolename): + raise NotImplementedError