Skip to content

Commit d488c2f

Browse files
m-novikovm-novikov
committed
Review fixes
1 parent ea64ec0 commit d488c2f

File tree

6 files changed

+166
-165
lines changed

6 files changed

+166
-165
lines changed

‎mypy/build.py

Lines changed: 107 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,28 @@ def __init__(self, manager: 'BuildManager', graph: Graph) -> None:
8585

8686

8787
class BuildSource:
88-
def __init__(self, path: Optional[str], module: Optional[str],
89-
text: Optional[str]) -> None:
90-
self.path = path
88+
def __init__(self, paths: Optional[Union[str, List[str]]], module: Optional[str],
89+
text: Optional[str], type: Optional['ModuleType'] = None) -> None:
90+
91+
if isinstance(paths, list):
92+
self.paths = paths
93+
elif paths is None:
94+
self.paths = []
95+
else:
96+
self.paths = [paths]
97+
9198
self.module = module or '__main__'
9299
self.text = text
100+
self.type = type
101+
102+
def __repr__(self) -> str:
103+
return 'BuildSource(%s)' % self.module
104+
105+
@property
106+
def path(self) -> Optional[str]:
107+
if self.paths:
108+
return self.paths[0]
109+
return None
93110

94111
def __repr__(self) -> str:
95112
return '<BuildSource path=%r module=%r has_text=%s>' % (self.path,
@@ -634,7 +651,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
634651

635652
return res
636653

637-
def find_module(self, id: str) -> Optional[str]:
654+
def find_module(self, id: str) -> Optional[BuildSource]:
638655
return self.module_discovery.find_module(id)
639656

640657
def is_module(self, id: str) -> bool:
@@ -837,12 +854,13 @@ def maybe_add_path(self, path: str, type: ModuleType) -> None:
837854
# But module can have multiple stubs
838855
py_path = path.endswith('.py')
839856
if self.has_py and py_path:
857+
# Found more than one implementation for module, skip it
840858
return None
841859

842860
if type == ModuleType.namespace:
843-
ok = self._verify_namespace(path)
861+
ok = os.path.isdir(path)
844862
else:
845-
ok = self._verify_module(path)
863+
ok = is_file(path)
846864

847865
if not ok:
848866
return None
@@ -853,50 +871,20 @@ def maybe_add_path(self, path: str, type: ModuleType) -> None:
853871
self.type = type
854872
self.paths.append(path)
855873

856-
def _verify_module(self, path: str) -> bool:
857-
# At this point we already know that that it's valid python path
858-
# We only need to check file existence
859-
if not is_file(path):
860-
return False
861-
862-
return True
863-
864-
def _verify_namespace(self, path: str) -> bool:
865-
if os.path.isdir(path):
866-
files = set(list_dir(path) or []) # type: Set[str]
867-
if not ('__init__.py' in files or '__init__.pyi' in files):
868-
return True
869-
870-
return False
871-
872-
873-
def is_pkg_path(path: str) -> bool:
874-
return path.endswith(('__init__.py', '__init__.pyi'))
875-
876-
877-
def is_module_path(path: str) -> bool:
878-
return not is_pkg_path(path) and path.endswith(('.py', '.pyi'))
879-
880-
881-
def is_namespace_path(path: str) -> bool:
882-
return path.endswith(os.path.sep)
883-
884874

885875
class ModuleDiscovery:
886876
def __init__(self,
887877
lib_path: Iterable[str],
888878
namespaces_allowed: bool = False) -> None:
889879

890-
self.lib_path = tuple(os.path.normpath(p) for p in lib_path) # type: Tuple[str, ...]
880+
self.lib_path = [os.path.normpath(p) for p in lib_path] # type: List[str]
891881
self.namespaces_allowed = namespaces_allowed
892-
self._find_module_cache = {} # type: Dict[str, List[str]]
882+
self._find_module_cache = {} # type: Dict[str, Optional[BuildSource]]
893883

894-
def find_module(self, id: str) -> Optional[str]:
895-
paths = self._find_module(id)
896-
if paths:
897-
return paths[0]
898-
else:
899-
return None
884+
def find_module(self, id: str) -> Optional[BuildSource]:
885+
if id not in self._find_module_cache:
886+
self._find_module_cache[id] = self._find_module(id)
887+
return self._find_module_cache[id]
900888

901889
def find_modules_recursive(self, module: str) -> List[BuildSource]:
902890
"""
@@ -911,81 +899,79 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
911899
result.append(src)
912900
return result
913901

902+
def _iter_module_dir_paths(self, source: Optional[BuildSource]) -> Iterator[str]:
903+
if not (source and source.paths):
904+
return
905+
906+
if source.type == ModuleType.package:
907+
if source.path:
908+
yield dirname(source.path)
909+
elif source.type == ModuleType.namespace:
910+
yield from source.paths
911+
914912
def _find_modules_recursive(self, module: str) -> List[BuildSource]:
915-
module_paths = self._find_module(module)
913+
src = self.find_module(module)
916914

917-
srcs = [] # type: List[BuildSource]
918-
for path in module_paths:
919-
if is_module_path(path) or is_pkg_path(path):
920-
srcs.append(BuildSource(path, module, None))
915+
if not src:
916+
return []
921917

922-
if is_pkg_path(path):
923-
path = dirname(path)
924-
for submodule in self._find_submodules(module, path):
925-
srcs += self._find_modules_recursive(submodule)
926-
elif is_namespace_path(path):
927-
for submodule in self._find_submodules(module, path):
928-
srcs += self._find_modules_recursive(submodule)
918+
srcs = [src] # type: List[BuildSource]
919+
for path in self._iter_module_dir_paths(src):
920+
for submodule in self._find_submodules(module, path):
921+
srcs += self._find_modules_recursive(submodule)
929922

930923
return srcs
931924

932925
def _find_submodules(self, module: str, path: str) -> Iterator[str]:
933926
for item in list_dir(path) or []:
934-
if item == '__init__.py' or item == '__init__.pyi':
927+
if item.startswith(('__', '.')):
935928
continue
936929

937930
if item.endswith(tuple(PYTHON_EXTENSIONS)):
938931
item = item.split('.')[0]
939932

940933
yield module + '.' + item
941934

942-
def _collect_paths(self, paths: List[str], last_comp: str) -> List[str]:
943-
"""
944-
Collect all available module paths
945-
"""
935+
def _find_module(self, id: str) -> Optional[BuildSource]:
936+
components = id.split('.')
937+
938+
if len(components) > 1:
939+
parent_id = '.'.join(components[:-1])
940+
parent = self.find_module(parent_id)
941+
if not parent:
942+
return None
943+
search_paths = list(self._iter_module_dir_paths(parent))
944+
else:
945+
search_paths = self.lib_path
946+
947+
leaf_module_name = components[-1]
946948
sepinit = '__init__'
947-
ctx = ImportContext()
948949

949950
# Detect modules in following order: package, module, namespace.
950951
# First hit determines module type, consistency of paths to given type
951952
# ensured in ImportContext
952-
for path_item in paths:
953-
if is_module_path(path_item):
954-
continue
955-
956-
if is_pkg_path(path_item):
957-
path_item = dirname(path_item)
958-
953+
for path in search_paths:
959954
for ext in PYTHON_EXTENSIONS:
960-
path = os.path.join(path_item, last_comp, sepinit + ext)
961-
ctx.maybe_add_path(path, ModuleType.package)
955+
candidate_path = os.path.join(path, leaf_module_name, sepinit + ext)
956+
if is_file(candidate_path):
957+
return BuildSource(candidate_path, id, None, type=ModuleType.package)
962958

963959
for ext in PYTHON_EXTENSIONS:
964-
path = os.path.join(path_item, last_comp + ext)
965-
ctx.maybe_add_path(path, ModuleType.module)
966-
967-
if self.namespaces_allowed and not ctx.paths:
968-
for path_item in paths:
969-
if is_pkg_path(path_item):
970-
path_item = dirname(path_item)
960+
candidate_path = os.path.join(path, leaf_module_name + ext)
961+
if is_file(candidate_path):
962+
return BuildSource(candidate_path, id, None, type=ModuleType.module)
971963

972-
path = os.path.join(path_item, last_comp)
973-
ctx.maybe_add_path(path + os.sep, ModuleType.namespace)
974-
975-
return ctx.paths
976-
977-
def _find_module(self, id: str) -> List[str]:
978-
if id not in self._find_module_cache:
979-
components = id.split('.')
980-
parent_paths = list(self.lib_path)
964+
if self.namespaces_allowed:
965+
namespace_paths = []
966+
for path in search_paths:
967+
candidate_path = os.path.join(path, leaf_module_name)
968+
if os.path.isdir(candidate_path):
969+
namespace_paths.append(candidate_path)
981970

982-
if len(components) > 1:
983-
parent_id = '.'.join(components[:-1])
984-
parent_paths = self._find_module(parent_id)
971+
if namespace_paths:
972+
return BuildSource(namespace_paths, id, None, type=ModuleType.namespace)
985973

986-
last_comp = components[-1]
987-
self._find_module_cache[id] = self._collect_paths(parent_paths, last_comp)
988-
return self._find_module_cache[id]
974+
return None
989975

990976

991977
def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]:
@@ -1617,36 +1603,36 @@ def __init__(self,
16171603
# difference and just assume 'builtins' everywhere,
16181604
# which simplifies code.
16191605
file_id = '__builtin__'
1620-
path = manager.find_module(file_id)
1621-
if path:
1622-
if not is_namespace_path(path):
1623-
# For non-stubs, look at options.follow_imports:
1624-
# - normal (default) -> fully analyze
1625-
# - silent -> analyze but silence errors
1626-
# - skip -> don't analyze, make the type Any
1627-
follow_imports = self.options.follow_imports
1628-
if (follow_imports != 'normal'
1629-
and not root_source # Honor top-level modules
1630-
and path.endswith('.py') # Stubs are always normal
1631-
and id != 'builtins'): # Builtins is always normal
1632-
if follow_imports == 'silent':
1633-
# Still import it, but silence non-blocker errors.
1634-
manager.log("Silencing %s (%s)" % (path, id))
1635-
self.ignore_all = True
1636-
else:
1637-
# In 'error' mode, produce special error messages.
1638-
manager.log("Skipping %s (%s)" % (path, id))
1639-
if follow_imports == 'error':
1640-
if ancestor_for:
1641-
self.skipping_ancestor(id, path, ancestor_for)
1642-
else:
1643-
self.skipping_module(id, path)
1644-
path = None
1645-
manager.missing_modules.add(id)
1646-
raise ModuleNotFound
1647-
else:
1648-
source = ''
1649-
self.source_hash = ''
1606+
src = manager.find_module(file_id)
1607+
if src and src.type == ModuleType.namespace:
1608+
source = ''
1609+
self.source_hash = ''
1610+
elif src and src.path:
1611+
path = src.path
1612+
# For non-stubs, look at options.follow_imports:
1613+
# - normal (default) -> fully analyze
1614+
# - silent -> analyze but silence errors
1615+
# - skip -> don't analyze, make the type Any
1616+
follow_imports = self.options.follow_imports
1617+
if (follow_imports != 'normal'
1618+
and not root_source # Honor top-level modules
1619+
and path.endswith('.py') # Stubs are always normal
1620+
and id != 'builtins'): # Builtins is always normal
1621+
if follow_imports == 'silent':
1622+
# Still import it, but silence non-blocker errors.
1623+
manager.log("Silencing %s (%s)" % (path, id))
1624+
self.ignore_all = True
1625+
else:
1626+
# In 'error' mode, produce special error messages.
1627+
manager.log("Skipping %s (%s)" % (path, id))
1628+
if follow_imports == 'error':
1629+
if ancestor_for:
1630+
self.skipping_ancestor(id, path, ancestor_for)
1631+
else:
1632+
self.skipping_module(id, path)
1633+
path = None
1634+
manager.missing_modules.add(id)
1635+
raise ModuleNotFound
16501636
else:
16511637
# Could not find a module. Typically the reason is a
16521638
# misspelled module name, missing stub, module not in

‎mypy/stubgen.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,11 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
157157
else:
158158
# Find module by going through search path.
159159
md = mypy.build.ModuleDiscovery(['.'] + search_path)
160-
module_path = md.find_module(module)
161-
if not module_path:
160+
src = md.find_module(module)
161+
if not (src and src.path):
162162
raise SystemExit(
163163
"Can't find module '{}' (consider using --search-path)".format(module))
164+
module_path = src.path
164165
module_all = None
165166
return module_path, module_all
166167

‎mypy/test/testcheck.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,11 +322,11 @@ def parse_module(self,
322322
out = []
323323
for module_name in module_names.split(' '):
324324
md = build.ModuleDiscovery([test_temp_dir], namespaces_allowed=False)
325-
path = md.find_module(module_name)
326-
assert path is not None, "Can't find ad hoc case file"
327-
with open(path) as f:
325+
src = md.find_module(module_name)
326+
assert src is not None and src.path is not None, "Can't find ad hoc case file"
327+
with open(src.path) as f:
328328
program_text = f.read()
329-
out.append((module_name, path, program_text))
329+
out.append((module_name, src.path, program_text))
330330
return out
331331
else:
332332
return [('__main__', 'main', program_text)]

‎mypy/test/testdmypy.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -254,11 +254,11 @@ def parse_module(self,
254254
out = []
255255
for module_name in module_names.split(' '):
256256
md = build.ModuleDiscovery([test_temp_dir])
257-
path = md.find_module(module_name)
258-
assert path is not None, "Can't find ad hoc case file"
259-
with open(path) as f:
257+
src = md.find_module(module_name)
258+
assert src is not None and src.path is not None, "Can't find ad hoc case file"
259+
with open(src.path) as f:
260260
program_text = f.read()
261-
out.append((module_name, path, program_text))
261+
out.append((module_name, src.path, program_text))
262262
return out
263263
else:
264264
return [('__main__', 'main', program_text)]

0 commit comments

Comments
 (0)