avoid race condition during chunk write

docs/release.rst

@@ -19,22 +19,28 @@ Enhancements
 * Support has been added for structured arrays with sub-array shape and/or nested fields. By
   :user:`Tarik Onalan <onalant>`, :issue:`111`, :issue:`296`.
 
-Maintenance
-~~~~~~~~~~~
+Bug fixes
+~~~~~~~~~
+
+* The implementation of the :class:`zarr.storage.DirectoryStore` class has been modified to 
+  ensure that writes are atomic and there are no race conditions where a chunk might appear 
+  transiently missing during a write operation. By :user:`sbalmer <sbalmer>`, :issue:`327`, 
+  :issue:`263`.
 
 * The required version of the `numcodecs <http://numcodecs.rtfd.io>`_ package has been upgraded 
   to 0.6.2, which has enabled some code simplification and fixes a failing test involving
   msgpack encoding. By :user:`John Kirkham <jakirkham>`, :issue:`352`, :issue:`355`, 
   :issue:`324`.
 
-* CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and
-  upgrade all pinned package requirements. :issue:`308`.
-
 * Failing tests related to pickling/unpickling have been fixed. By :user:`Ryan Williams <ryan-williams>`,
   :issue:`273`, :issue:`308`.
 
-Acknowledgments
-~~~~~~~~~~~~~~~
+Maintenance
+~~~~~~~~~~~
+
+* CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and
+  upgrade all pinned package requirements. :issue:`308`.
+
 
 .. _release_2.2.0:
 

requirements.txt

@@ -3,3 +3,4 @@ fasteners
 numcodecs
 numpy
 pytest
+pyosreplace; python_version < '3.3' and sys.platform == 'win32'

requirements_dev.txt

@@ -1,3 +1,4 @@
 asciitree==0.3.3
 fasteners==0.14.1
 numcodecs==0.6.2
+pyosreplace==0.1; python_version < '3.3' and sys.platform == 'win32'

setup.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 from setuptools import setup
+import sys
 
 
 DESCRIPTION = 'An implementation of chunked, compressed, ' \
@@ -9,6 +10,16 @@
 with open('README.rst') as f:
     LONG_DESCRIPTION = f.read()
 
+dependencies = [
+    'asciitree',
+    'numpy>=1.7',
+    'fasteners',
+    'numcodecs>=0.6.2',
+]
+
+if sys.version_info < (3, 3) and sys.platform == "win32":
+    dependencies.append('pyosreplace')
+
 setup(
     name='zarr',
     description=DESCRIPTION,
@@ -22,12 +33,7 @@
         'setuptools>18.0',
         'setuptools-scm>1.5.4'
     ],
-    install_requires=[
-        'asciitree',
-        'numpy>=1.7',
-        'fasteners',
-        'numcodecs>=0.6.2',
-    ],
+    install_requires=dependencies,
     package_dir={'': '.'},
     packages=['zarr', 'zarr.tests'],
     classifiers=[
@@ -42,7 +48,6 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

zarr/storage.py

@@ -53,6 +53,15 @@
     from zarr.codecs import Zlib
     default_compressor = Zlib()
 
+# Find which function to use for atomic replace
+if sys.version_info >= (3, 3):
+    from os import replace
+elif sys.platform == "win32":  # pragma: no cover
+    from osreplace import replace
+else:  # pragma: no cover
+    # POSIX rename() is always atomic
+    from os import rename as replace
+
 
 def _path_to_prefix(path):
     # assume path already normalized
@@ -752,9 +761,7 @@ def __setitem__(self, key, value):
                 f.write(value)
 
             # move temporary file into place
-            if os.path.exists(file_path):
-                os.remove(file_path)
-            os.rename(temp_path, file_path)
+            replace(temp_path, file_path)
 
         finally:
             # clean up if temp file still exists for whatever reason