file storage

Author: ewdurbin

docker-compose.yml

@@ -1,6 +1,7 @@
 version: '3'
 
 volumes:
+  simple:
   packages:
   vault:
 
@@ -85,6 +86,7 @@ services:
       - ./htmlcov:/opt/warehouse/src/htmlcov:z
       - .coveragerc:/opt/warehouse/src/.coveragerc:z
       - packages:/var/opt/warehouse/packages
+      - simple:/var/opt/warehouse/simple
     ports:
       - "80:8000"
 
@@ -95,6 +97,7 @@ services:
     command: python -m http.server 9001
     volumes:
       - packages:/var/opt/warehouse/packages
+      - simple:/var/opt/warehouse/simple
     ports:
       - "9001:9001"
 
@@ -110,6 +113,7 @@ services:
     environment:
       C_FORCE_ROOT: "1"
       FILES_BACKEND: "warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://files:9001/packages/{path}"
+      SIMPLE_BACKEND: "warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}"
 
   static:
     build:

warehouse/packaging/interfaces.py

@@ -34,6 +34,27 @@ def store(path, file_path, *, meta=None):
         """
 
 
+class ISimpleStorage(Interface):
+    def create_service(context, request):
+        """
+        Create the service, given the context and request for which it is being
+        created for, passing a name for settings.
+        """
+
+    def get(path):
+        """
+        Return a file like object that can be read to access the file located
+        at the given path.
+        """
+
+    def store(path, file_path, *, meta=None):
+        """
+        Save the file located at file_path to the file storage at the location
+        specified by path. An additional meta keyword argument may contain
+        extra information that an implementation may or may not store.
+        """
+
+
 class IDocsStorage(Interface):
     def create_service(context, request):
         """

warehouse/packaging/services.py

@@ -20,7 +20,7 @@
 
 from zope.interface import implementer
 
-from warehouse.packaging.interfaces import IDocsStorage, IFileStorage
+from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage
 
 
 class InsecureStorageWarning(UserWarning):
@@ -137,6 +137,37 @@ def store(self, path, file_path, *, meta=None):
         self.bucket.upload_file(file_path, path, ExtraArgs=extra_args)
 
 
+@implementer(ISimpleStorage)
+class S3SimpleStorage(GenericFileStorage):
+    @classmethod
+    def create_service(cls, context, request):
+        session = request.find_service(name="aws.session")
+        s3 = session.resource("s3")
+        bucket = s3.Bucket(request.registry.settings["files.bucket"])
+        prefix = request.registry.settings.get("files.prefix")
+        return cls(bucket, prefix=prefix)
+
+    def get(self, path):
+        # Note: this is not actually used in production, instead our CDN is
+        # configured to connect directly to our storage bucket. See:
+        # https://github.com/python/pypi-infra/blob/master/terraform/file-hosting/vcl/main.vcl
+        try:
+            return self.bucket.Object(self._get_path(path)).get()["Body"]
+        except botocore.exceptions.ClientError as exc:
+            if exc.response["Error"]["Code"] != "NoSuchKey":
+                raise
+            raise FileNotFoundError("No such key: {!r}".format(path)) from None
+
+    def store(self, path, file_path, *, meta=None):
+        extra_args = {}
+        if meta is not None:
+            extra_args["Metadata"] = meta
+
+        path = self._get_path(path)
+
+        self.bucket.upload_file(file_path, path, ExtraArgs=extra_args)
+
+
 @implementer(IDocsStorage)
 class S3DocsStorage:
     def __init__(self, s3_client, bucket_name, *, prefix=None):
@@ -203,3 +234,38 @@ def store(self, path, file_path, *, meta=None):
         if meta is not None:
             blob.metadata = meta
         blob.upload_from_filename(file_path)
+
+
+@implementer(ISimpleStorage)
+class GCSSimpleStorage(GenericFileStorage):
+    @classmethod
+    @google.api_core.retry.Retry(
+        predicate=google.api_core.retry.if_exception_type(
+            google.api_core.exceptions.ServiceUnavailable
+        )
+    )
+    def create_service(cls, context, request):
+        storage_client = request.find_service(name="gcloud.gcs")
+        bucket_name = request.registry.settings["files.bucket"]
+        bucket = storage_client.get_bucket(bucket_name)
+        prefix = request.registry.settings.get("files.prefix")
+
+        return cls(bucket, prefix=prefix)
+
+    def get(self, path):
+        # Note: this is not actually used in production, instead our CDN is
+        # configured to connect directly to our storage bucket. See:
+        # https://github.com/python/pypi-infra/blob/master/terraform/file-hosting/vcl/main.vcl
+        raise NotImplementedError
+
+    @google.api_core.retry.Retry(
+        predicate=google.api_core.retry.if_exception_type(
+            google.api_core.exceptions.ServiceUnavailable
+        )
+    )
+    def store(self, path, file_path, *, meta=None):
+        path = self._get_path(path)
+        blob = self.bucket.blob(path)
+        if meta is not None:
+            blob.metadata = meta
+        blob.upload_from_filename(file_path)

warehouse/packaging/utils.py

@@ -1,10 +1,7 @@
 import hashlib
-import os.path
 
 from pyramid_jinja2 import IJinja2Environment
 
-import warehouse
-
 from warehouse.legacy.api.simple import _simple_detail
 
 
@@ -22,7 +19,8 @@ def render_simple_detail(project, request, store=False):
 
     if store:
         #  TODO: Store generated file in FileStorage
-        #        We should probably configure a new FileStorage for a new simple-files bucket in GCS
+        #        We should probably configure a new FileStorage
+        #        for a new simple-files bucket in GCS
         pass
 
     return (content_hash, simple_detail_path)