Skip to content

Commit 17d5b9d

Browse files
gh-59110: zipimport: support namespace packages when no directory entry exists (GH-121233)
1 parent db17291 commit 17d5b9d

File tree

4 files changed

+140
-30
lines changed

4 files changed

+140
-30
lines changed

Lib/test/test_importlib/test_namespace_pkgs.py

+13-14
Original file line numberDiff line numberDiff line change
@@ -286,25 +286,24 @@ def test_project3_succeeds(self):
286286

287287
class ZipWithMissingDirectory(NamespacePackageTest):
288288
paths = ['missing_directory.zip']
289+
# missing_directory.zip contains:
290+
# Length Date Time Name
291+
# --------- ---------- ----- ----
292+
# 29 2012-05-03 18:13 foo/one.py
293+
# 0 2012-05-03 20:57 bar/
294+
# 38 2012-05-03 20:57 bar/two.py
295+
# --------- -------
296+
# 67 3 files
289297

290-
@unittest.expectedFailure
291298
def test_missing_directory(self):
292-
# This will fail because missing_directory.zip contains:
293-
# Length Date Time Name
294-
# --------- ---------- ----- ----
295-
# 29 2012-05-03 18:13 foo/one.py
296-
# 0 2012-05-03 20:57 bar/
297-
# 38 2012-05-03 20:57 bar/two.py
298-
# --------- -------
299-
# 67 3 files
300-
301-
# Because there is no 'foo/', the zipimporter currently doesn't
302-
# know that foo is a namespace package
303-
304299
import foo.one
300+
self.assertEqual(foo.one.attr, 'portion1 foo one')
301+
302+
def test_missing_directory2(self):
303+
import foo
304+
self.assertFalse(hasattr(foo, 'one'))
305305

306306
def test_present_directory(self):
307-
# This succeeds because there is a "bar/" in the zip file
308307
import bar.two
309308
self.assertEqual(bar.two.attr, 'missing_directory foo two')
310309

Lib/test/test_zipimport.py

+109-16
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,81 @@ def testSubNamespacePackage(self):
296296
packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)}
297297
self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD)
298298

299+
def testPackageExplicitDirectories(self):
300+
# Test explicit namespace packages with explicit directory entries.
301+
self.addCleanup(os_helper.unlink, TEMP_ZIP)
302+
with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
303+
z.mkdir('a')
304+
z.writestr('a/__init__.py', test_src)
305+
z.mkdir('a/b')
306+
z.writestr('a/b/__init__.py', test_src)
307+
z.mkdir('a/b/c')
308+
z.writestr('a/b/c/__init__.py', test_src)
309+
z.writestr('a/b/c/d.py', test_src)
310+
self._testPackage(initfile='__init__.py')
311+
312+
def testPackageImplicitDirectories(self):
313+
# Test explicit namespace packages without explicit directory entries.
314+
self.addCleanup(os_helper.unlink, TEMP_ZIP)
315+
with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
316+
z.writestr('a/__init__.py', test_src)
317+
z.writestr('a/b/__init__.py', test_src)
318+
z.writestr('a/b/c/__init__.py', test_src)
319+
z.writestr('a/b/c/d.py', test_src)
320+
self._testPackage(initfile='__init__.py')
321+
322+
def testNamespacePackageExplicitDirectories(self):
323+
# Test implicit namespace packages with explicit directory entries.
324+
self.addCleanup(os_helper.unlink, TEMP_ZIP)
325+
with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
326+
z.mkdir('a')
327+
z.mkdir('a/b')
328+
z.mkdir('a/b/c')
329+
z.writestr('a/b/c/d.py', test_src)
330+
self._testPackage(initfile=None)
331+
332+
def testNamespacePackageImplicitDirectories(self):
333+
# Test implicit namespace packages without explicit directory entries.
334+
self.addCleanup(os_helper.unlink, TEMP_ZIP)
335+
with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
336+
z.writestr('a/b/c/d.py', test_src)
337+
self._testPackage(initfile=None)
338+
339+
def _testPackage(self, initfile):
340+
zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'a'))
341+
if initfile is None:
342+
# XXX Should it work?
343+
self.assertRaises(zipimport.ZipImportError, zi.is_package, 'b')
344+
self.assertRaises(zipimport.ZipImportError, zi.get_source, 'b')
345+
self.assertRaises(zipimport.ZipImportError, zi.get_code, 'b')
346+
else:
347+
self.assertTrue(zi.is_package('b'))
348+
self.assertEqual(zi.get_source('b'), test_src)
349+
self.assertEqual(zi.get_code('b').co_filename,
350+
os.path.join(TEMP_ZIP, 'a', 'b', initfile))
351+
352+
sys.path.insert(0, TEMP_ZIP)
353+
self.assertNotIn('a', sys.modules)
354+
355+
mod = importlib.import_module(f'a.b')
356+
self.assertIn('a', sys.modules)
357+
self.assertIs(sys.modules['a.b'], mod)
358+
if initfile is None:
359+
self.assertIsNone(mod.__file__)
360+
else:
361+
self.assertEqual(mod.__file__,
362+
os.path.join(TEMP_ZIP, 'a', 'b', initfile))
363+
self.assertEqual(len(mod.__path__), 1, mod.__path__)
364+
self.assertEqual(mod.__path__[0], os.path.join(TEMP_ZIP, 'a', 'b'))
365+
366+
mod2 = importlib.import_module(f'a.b.c.d')
367+
self.assertIn('a.b.c', sys.modules)
368+
self.assertIn('a.b.c.d', sys.modules)
369+
self.assertIs(sys.modules['a.b.c.d'], mod2)
370+
self.assertIs(mod.c.d, mod2)
371+
self.assertEqual(mod2.__file__,
372+
os.path.join(TEMP_ZIP, 'a', 'b', 'c', 'd.py'))
373+
299374
def testMixedNamespacePackage(self):
300375
# Test implicit namespace packages spread between a
301376
# real filesystem and a zip archive.
@@ -520,6 +595,7 @@ def testInvalidateCaches(self):
520595
packdir2 + "__init__" + pyc_ext: (NOW, test_pyc),
521596
packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc),
522597
"spam" + pyc_ext: (NOW, test_pyc)}
598+
extra_files = [packdir, packdir2]
523599
self.addCleanup(os_helper.unlink, TEMP_ZIP)
524600
with ZipFile(TEMP_ZIP, "w") as z:
525601
for name, (mtime, data) in files.items():
@@ -529,10 +605,10 @@ def testInvalidateCaches(self):
529605
z.writestr(zinfo, data)
530606

531607
zi = zipimport.zipimporter(TEMP_ZIP)
532-
self.assertEqual(zi._get_files().keys(), files.keys())
608+
self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
533609
# Check that the file information remains accurate after reloading
534610
zi.invalidate_caches()
535-
self.assertEqual(zi._get_files().keys(), files.keys())
611+
self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
536612
# Add a new file to the ZIP archive
537613
newfile = {"spam2" + pyc_ext: (NOW, test_pyc)}
538614
files.update(newfile)
@@ -544,7 +620,7 @@ def testInvalidateCaches(self):
544620
z.writestr(zinfo, data)
545621
# Check that we can detect the new file after invalidating the cache
546622
zi.invalidate_caches()
547-
self.assertEqual(zi._get_files().keys(), files.keys())
623+
self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
548624
spec = zi.find_spec('spam2')
549625
self.assertIsNotNone(spec)
550626
self.assertIsInstance(spec.loader, zipimport.zipimporter)
@@ -562,6 +638,7 @@ def testInvalidateCachesWithMultipleZipimports(self):
562638
packdir2 + "__init__" + pyc_ext: (NOW, test_pyc),
563639
packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc),
564640
"spam" + pyc_ext: (NOW, test_pyc)}
641+
extra_files = [packdir, packdir2]
565642
self.addCleanup(os_helper.unlink, TEMP_ZIP)
566643
with ZipFile(TEMP_ZIP, "w") as z:
567644
for name, (mtime, data) in files.items():
@@ -571,10 +648,10 @@ def testInvalidateCachesWithMultipleZipimports(self):
571648
z.writestr(zinfo, data)
572649

573650
zi = zipimport.zipimporter(TEMP_ZIP)
574-
self.assertEqual(zi._get_files().keys(), files.keys())
651+
self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
575652
# Zipimporter for the same path.
576653
zi2 = zipimport.zipimporter(TEMP_ZIP)
577-
self.assertEqual(zi2._get_files().keys(), files.keys())
654+
self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files]))
578655
# Add a new file to the ZIP archive to make the cache wrong.
579656
newfile = {"spam2" + pyc_ext: (NOW, test_pyc)}
580657
files.update(newfile)
@@ -587,7 +664,7 @@ def testInvalidateCachesWithMultipleZipimports(self):
587664
# Invalidate the cache of the first zipimporter.
588665
zi.invalidate_caches()
589666
# Check that the second zipimporter detects the new file and isn't using a stale cache.
590-
self.assertEqual(zi2._get_files().keys(), files.keys())
667+
self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files]))
591668
spec = zi2.find_spec('spam2')
592669
self.assertIsNotNone(spec)
593670
self.assertIsInstance(spec.loader, zipimport.zipimporter)
@@ -650,17 +727,33 @@ def testZipImporterMethodsInSubDirectory(self):
650727
self.assertIsNone(loader.get_source(mod_name))
651728
self.assertEqual(loader.get_filename(mod_name), mod.__file__)
652729

653-
def testGetData(self):
730+
def testGetDataExplicitDirectories(self):
654731
self.addCleanup(os_helper.unlink, TEMP_ZIP)
655-
with ZipFile(TEMP_ZIP, "w") as z:
656-
z.compression = self.compression
657-
name = "testdata.dat"
658-
data = bytes(x for x in range(256))
659-
z.writestr(name, data)
660-
661-
zi = zipimport.zipimporter(TEMP_ZIP)
662-
self.assertEqual(data, zi.get_data(name))
663-
self.assertIn('zipimporter object', repr(zi))
732+
with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
733+
z.mkdir('a')
734+
z.mkdir('a/b')
735+
z.mkdir('a/b/c')
736+
data = bytes(range(256))
737+
z.writestr('a/b/c/testdata.dat', data)
738+
self._testGetData()
739+
740+
def testGetDataImplicitDirectories(self):
741+
self.addCleanup(os_helper.unlink, TEMP_ZIP)
742+
with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
743+
data = bytes(range(256))
744+
z.writestr('a/b/c/testdata.dat', data)
745+
self._testGetData()
746+
747+
def _testGetData(self):
748+
zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'ignored'))
749+
pathname = os.path.join('a', 'b', 'c', 'testdata.dat')
750+
data = bytes(range(256))
751+
self.assertEqual(zi.get_data(pathname), data)
752+
self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, pathname)), data)
753+
self.assertEqual(zi.get_data(os.path.join('a', 'b', '')), b'')
754+
self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, 'a', 'b', '')), b'')
755+
self.assertRaises(OSError, zi.get_data, os.path.join('a', 'b'))
756+
self.assertRaises(OSError, zi.get_data, os.path.join(TEMP_ZIP, 'a', 'b'))
664757

665758
def testImporterAttr(self):
666759
src = """if 1: # indent hack

Lib/zipimport.py

+16
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ def get_data(self, pathname):
155155
toc_entry = self._get_files()[key]
156156
except KeyError:
157157
raise OSError(0, '', key)
158+
if toc_entry is None:
159+
return b''
158160
return _get_data(self.archive, toc_entry)
159161

160162

@@ -554,6 +556,20 @@ def _read_directory(archive):
554556
finally:
555557
fp.seek(start_offset)
556558
_bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive)
559+
560+
# Add implicit directories.
561+
for name in list(files):
562+
while True:
563+
i = name.rstrip(path_sep).rfind(path_sep)
564+
if i < 0:
565+
break
566+
name = name[:i + 1]
567+
if name in files:
568+
break
569+
files[name] = None
570+
count += 1
571+
_bootstrap._verbose_message('zipimport: added {} implicit directories in {!r}',
572+
count, archive)
557573
return files
558574

559575
# During bootstrap, we may need to load the encodings
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
:mod:`zipimport` supports now namespace packages when no directory entry
2+
exists.

0 commit comments

Comments
 (0)