Handle non-existent bucket for s3 mapping

Author: mrocklin

s3fs/core.py

@@ -463,6 +463,8 @@ def bulk_delete(self, pathlist):
         pathlist : listof strings
             The keys to remove, must all be in the same bucket.
         """
+        if not pathlist:
+            return
         buckets = {split_path(path)[0] for path in pathlist}
         if len(buckets) > 1:
             raise ValueError("Bulk delete files should refer to only one bucket")

s3fs/mapping.py

@@ -2,7 +2,7 @@
 from collections import MutableMapping
 import os
 
-from .core import S3FileSystem
+from .core import S3FileSystem, split_path
 
 
 class S3Map(MutableMapping):
@@ -30,12 +30,19 @@ class S3Map(MutableMapping):
     b'Hello World'
     """
 
-    def __init__(self, root, s3=None, check=False):
+    def __init__(self, root, s3=None, check=False, create=False):
         self.s3 = s3 or S3FileSystem.current()
         self.root = root
         if check:
             self.s3.touch(root+'/a')
             self.s3.rm(root+'/a')
+        else:
+            bucket = split_path(root)[0]
+            if create:
+                self.s3.mkdir(bucket)
+            elif not self.s3.exists(bucket):
+                raise ValueError("Bucket %s does not exist."
+                        " Create bucket with the ``create=True`` keyword")
 
     def clear(self):
         """Remove all keys below root - empties out mapping

s3fs/tests/test_mapping.py

@@ -91,3 +91,17 @@ def test_bytearray(s3):
     d['x'] = bytearray(b'123')
 
     assert d['x'] == b'123'
+
+
+def test_new_bucket(s3):
+    try:
+        d = S3Map('new-bucket', s3)
+        assert False
+    except ValueError as e:
+        assert 'create=True' in str(e)
+
+    d = S3Map('new-bucket', s3, create=True)
+    assert not d
+
+    d = S3Map('new-bucket/new-directory', s3)
+    assert not d