file storage

Author: ewdurbin

dev/environment

@@ -23,6 +23,7 @@ CAMO_KEY=insecurecamokey
 DOCS_URL="https://pythonhosted.org/{project}/"
 
 FILES_BACKEND=warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://localhost:9001/packages/{path}
+SIMPLE_BACKEND=warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://localhost:9001/simple/{path}
 DOCS_BACKEND=warehouse.packaging.services.LocalDocsStorage path=/var/opt/warehouse/docs/
 
 MAIL_BACKEND=warehouse.email.services.SMTPEmailSender host=smtp port=2525 ssl=false [email protected]

docker-compose.yml

@@ -1,6 +1,7 @@
 version: '3'
 
 volumes:
+  simple:
   packages:
   vault:
 
@@ -85,6 +86,7 @@ services:
       - ./htmlcov:/opt/warehouse/src/htmlcov:z
       - .coveragerc:/opt/warehouse/src/.coveragerc:z
       - packages:/var/opt/warehouse/packages
+      - simple:/var/opt/warehouse/simple
       - ./bin:/opt/warehouse/src/bin:z
     ports:
       - "80:8000"
@@ -96,6 +98,7 @@ services:
     command: python -m http.server 9001
     volumes:
       - packages:/var/opt/warehouse/packages
+      - simple:/var/opt/warehouse/simple
     ports:
       - "9001:9001"
 
@@ -111,6 +114,7 @@ services:
     environment:
       C_FORCE_ROOT: "1"
       FILES_BACKEND: "warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://files:9001/packages/{path}"
+      SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}"
 
   static:
     build:

warehouse/config.py

@@ -216,6 +216,7 @@ def configure(settings=None):
         default=21600,  # 6 hours
     )
     maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
+    maybe_set_compound(settings, "simple", "backend", "SIMPLE_BACKEND")
     maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND")
     maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
     maybe_set_compound(settings, "mail", "backend", "MAIL_BACKEND")

warehouse/legacy/api/simple.py

@@ -11,15 +11,14 @@
 # limitations under the License.
 
 
-from packaging.version import parse
 from pyramid.httpexceptions import HTTPMovedPermanently
 from pyramid.view import view_config
 from sqlalchemy import func
-from sqlalchemy.orm import joinedload
 
 from warehouse.cache.http import cache_control
 from warehouse.cache.origin import origin_cache
-from warehouse.packaging.models import File, JournalEntry, Project, Release
+from warehouse.packaging.models import JournalEntry, Project
+from warehouse.packaging.utils import _simple_detail, render_simple_detail
 
 
 @view_config(
@@ -49,20 +48,6 @@ def simple_index(request):
     return {"projects": projects}
 
 
-def _simple_detail(project, request):
-    # Get all of the files for this project.
-    files = sorted(
-        request.db.query(File)
-        .options(joinedload(File.release))
-        .join(Release)
-        .filter(Release.project == project)
-        .all(),
-        key=lambda f: (parse(f.release.version), f.filename),
-    )
-
-    return {"project": project, "files": files}
-
-
 @view_config(
     route_name="legacy.api.simple.detail",
     context=Project,
@@ -88,4 +73,6 @@ def simple_detail(project, request):
     # Get the latest serial number for this project.
     request.response.headers["X-PyPI-Last-Serial"] = str(project.last_serial)
 
+    render_simple_detail(project, request, store=True)
+
     return _simple_detail(project, request)

warehouse/packaging/__init__.py

@@ -17,7 +17,7 @@
 from warehouse.accounts.models import Email, User
 from warehouse.cache.origin import key_factory, receive_set
 from warehouse.manage.tasks import update_role_invitation_status
-from warehouse.packaging.interfaces import IDocsStorage, IFileStorage
+from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage
 from warehouse.packaging.models import File, Project, Release, Role
 from warehouse.packaging.tasks import (
     compute_trending,
@@ -44,6 +44,9 @@ def includeme(config):
     files_storage_class = config.maybe_dotted(config.registry.settings["files.backend"])
     config.register_service_factory(files_storage_class.create_service, IFileStorage)
 
+    simple_storage_class = config.maybe_dotted(config.registry.settings["simple.backend"])
+    config.register_service_factory(simple_storage_class.create_service, ISimpleStorage)
+
     docs_storage_class = config.maybe_dotted(config.registry.settings["docs.backend"])
     config.register_service_factory(docs_storage_class.create_service, IDocsStorage)
 

warehouse/packaging/interfaces.py

@@ -34,6 +34,27 @@ def store(path, file_path, *, meta=None):
         """
 
 
+class ISimpleStorage(Interface):
+    def create_service(context, request):
+        """
+        Create the service, given the context and request for which it is being
+        created for, passing a name for settings.
+        """
+
+    def get(path):
+        """
+        Return a file like object that can be read to access the file located
+        at the given path.
+        """
+
+    def store(path, file_path, *, meta=None):
+        """
+        Save the file located at file_path to the file storage at the location
+        specified by path. An additional meta keyword argument may contain
+        extra information that an implementation may or may not store.
+        """
+
+
 class IDocsStorage(Interface):
     def create_service(context, request):
         """

warehouse/packaging/services.py

@@ -20,7 +20,7 @@
 
 from zope.interface import implementer
 
-from warehouse.packaging.interfaces import IDocsStorage, IFileStorage
+from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage
 
 
 class InsecureStorageWarning(UserWarning):
@@ -58,6 +58,37 @@ def store(self, path, file_path, *, meta=None):
                 dest_fp.write(src_fp.read())
 
 
+@implementer(ISimpleStorage)
+class LocalSimpleStorage:
+    def __init__(self, base):
+        # This class should not be used in production, it's trivial for it to
+        # be used to read arbitrary files from the disk. It is intended ONLY
+        # for local development with trusted users. To make this clear, we'll
+        # raise a warning.
+        warnings.warn(
+            "LocalSimpleStorage is intended only for use in development, you "
+            "should not use it in production due to the lack of safe guards "
+            "for safely locating files on disk.",
+            InsecureStorageWarning,
+        )
+
+        self.base = base
+
+    @classmethod
+    def create_service(cls, context, request):
+        return cls(request.registry.settings["simple.path"])
+
+    def get(self, path):
+        return open(os.path.join(self.base, path), "rb")
+
+    def store(self, path, file_path, *, meta=None):
+        destination = os.path.join(self.base, path)
+        print(destination)
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
+        with open(destination, "wb") as dest_fp:
+            with open(file_path, "rb") as src_fp:
+                dest_fp.write(src_fp.read())
+
 @implementer(IDocsStorage)
 class LocalDocsStorage:
     def __init__(self, base):
@@ -137,6 +168,37 @@ def store(self, path, file_path, *, meta=None):
         self.bucket.upload_file(file_path, path, ExtraArgs=extra_args)
 
 
+@implementer(ISimpleStorage)
+class S3SimpleStorage(GenericFileStorage):
+    @classmethod
+    def create_service(cls, context, request):
+        session = request.find_service(name="aws.session")
+        s3 = session.resource("s3")
+        bucket = s3.Bucket(request.registry.settings["files.bucket"])
+        prefix = request.registry.settings.get("files.prefix")
+        return cls(bucket, prefix=prefix)
+
+    def get(self, path):
+        # Note: this is not actually used in production, instead our CDN is
+        # configured to connect directly to our storage bucket. See:
+        # https://github.com/python/pypi-infra/blob/master/terraform/file-hosting/vcl/main.vcl
+        try:
+            return self.bucket.Object(self._get_path(path)).get()["Body"]
+        except botocore.exceptions.ClientError as exc:
+            if exc.response["Error"]["Code"] != "NoSuchKey":
+                raise
+            raise FileNotFoundError("No such key: {!r}".format(path)) from None
+
+    def store(self, path, file_path, *, meta=None):
+        extra_args = {}
+        if meta is not None:
+            extra_args["Metadata"] = meta
+
+        path = self._get_path(path)
+
+        self.bucket.upload_file(file_path, path, ExtraArgs=extra_args)
+
+
 @implementer(IDocsStorage)
 class S3DocsStorage:
     def __init__(self, s3_client, bucket_name, *, prefix=None):
@@ -203,3 +265,38 @@ def store(self, path, file_path, *, meta=None):
         if meta is not None:
             blob.metadata = meta
         blob.upload_from_filename(file_path)
+
+
+@implementer(ISimpleStorage)
+class GCSSimpleStorage(GenericFileStorage):
+    @classmethod
+    @google.api_core.retry.Retry(
+        predicate=google.api_core.retry.if_exception_type(
+            google.api_core.exceptions.ServiceUnavailable
+        )
+    )
+    def create_service(cls, context, request):
+        storage_client = request.find_service(name="gcloud.gcs")
+        bucket_name = request.registry.settings["files.bucket"]
+        bucket = storage_client.get_bucket(bucket_name)
+        prefix = request.registry.settings.get("files.prefix")
+
+        return cls(bucket, prefix=prefix)
+
+    def get(self, path):
+        # Note: this is not actually used in production, instead our CDN is
+        # configured to connect directly to our storage bucket. See:
+        # https://github.com/python/pypi-infra/blob/master/terraform/file-hosting/vcl/main.vcl
+        raise NotImplementedError
+
+    @google.api_core.retry.Retry(
+        predicate=google.api_core.retry.if_exception_type(
+            google.api_core.exceptions.ServiceUnavailable
+        )
+    )
+    def store(self, path, file_path, *, meta=None):
+        path = self._get_path(path)
+        blob = self.bucket.blob(path)
+        if meta is not None:
+            blob.metadata = meta
+        blob.upload_from_filename(file_path)

warehouse/packaging/utils.py

@@ -1,28 +1,51 @@
 import hashlib
-import os.path
+import tempfile
 
+from packaging.version import parse
 from pyramid_jinja2 import IJinja2Environment
+from sqlalchemy.orm import joinedload
 
-import warehouse
+from warehouse.packaging.interfaces import ISimpleStorage
+from warehouse.packaging.models import File, Release
 
-from warehouse.legacy.api.simple import _simple_detail
+
+def _simple_detail(project, request):
+   # Get all of the files for this project.
+    files = sorted(
+        request.db.query(File)
+        .options(joinedload(File.release))
+        .join(Release)
+        .filter(Release.project == project)
+        .all(),
+        key=lambda f: (parse(f.release.version), f.filename),
+    )
+
+    return {"project": project, "files": files}
 
 
 def render_simple_detail(project, request, store=False):
     context = _simple_detail(project, request)
 
     env = request.registry.queryUtility(IJinja2Environment, name=".jinja2")
-    template = env.get_template("legacy/api/simple/detail.html")
+    template = env.get_template("templates/legacy/api/simple/detail.html")
     content = template.render(**context, request=request)
 
     content_hasher = hashlib.blake2b(digest_size=256 // 8)
     content_hasher.update(content.encode("utf-8"))
     content_hash = content_hasher.hexdigest().lower()
-    simple_detail_path = f"/simple/{project.normalized_name}/{content_hash}/"
+    simple_detail_path = f"{project.normalized_name}/{content_hash}.html"
 
     if store:
-        #  TODO: Store generated file in FileStorage
-        #        We should probably configure a new FileStorage for a new simple-files bucket in GCS
-        pass
+        storage = request.find_service(ISimpleStorage)
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(content.encode('utf-8'))
+            storage.store(
+                simple_detail_path,
+                f.name,
+                meta={
+                    "project": project.normalized_name,
+                    "hash": content_hash,
+                },
+            )
 
     return (content_hash, simple_detail_path)