Skip to content

Always write the cache (unless cache_dir is /dev/null) #3133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Apr 19, 2017
Merged

Always write the cache (unless cache_dir is /dev/null) #3133

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b3578a5
Always write the cache (unless cache_dir is /dev/null)
Apr 4, 2017
3f4706c
Respond to review; move get_stat(path) up and be more robust
Apr 5, 2017
1338bca
Fix format string
Apr 19, 2017
49a6665
Merge branch 'master' into always-write-cache
Apr 19, 2017
114be68
Handle a rare race condition with makedirs().
Apr 19, 2017
c836125
Better fix for makedirs() race
Apr 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 35 additions & 24 deletions mypy/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -860,8 +860,7 @@ def write_cache(id: str, path: str, tree: MypyFile,

# Make sure directory for cache files exists
parent = os.path.dirname(data_json)
if not os.path.isdir(parent):
os.makedirs(parent)
os.makedirs(parent, exist_ok=True)
assert os.path.dirname(meta_json) == parent

# Construct temp file names
Expand All @@ -877,6 +876,20 @@ def write_cache(id: str, path: str, tree: MypyFile,
data_str = json.dumps(data, sort_keys=True)
interface_hash = compute_hash(data_str)

# Obtain and set up metadata
try:
st = manager.get_stat(path)
except OSError as err:
manager.log("Cannot get stat for {}: {}".format(path, err))
# Remove apparently-invalid cache files.
for filename in [data_json, meta_json]:
try:
os.remove(filename)
except OSError:
pass
# Still return the interface hash we computed.
return interface_hash

# Write data cache file, if applicable
if old_interface_hash == interface_hash:
# If the interface is unchanged, the cached data is guaranteed
Expand All @@ -891,8 +904,6 @@ def write_cache(id: str, path: str, tree: MypyFile,
os.replace(data_json_tmp, data_json)
manager.trace("Interface for {} has changed".format(id))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't the replace in :890 be only done if there's no OSError in :895?


# Obtain and set up metadata
st = manager.get_stat(path) # TODO: Handle errors
mtime = st.st_mtime
size = st.st_size
options = manager.options.clone_for_module(id)
Expand Down Expand Up @@ -1524,26 +1535,26 @@ def valid_references(self) -> Set[str]:
return valid_refs

def write_cache(self) -> None:
ok = self.path and self.options.incremental
if ok:
if self.manager.options.quick_and_dirty:
is_errors = self.manager.errors.is_errors_for_file(self.path)
else:
is_errors = self.manager.errors.is_errors()
ok = not is_errors
if ok:
dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies]
new_interface_hash = write_cache(
self.id, self.path, self.tree,
list(self.dependencies), list(self.suppressed), list(self.child_modules),
dep_prios, self.interface_hash,
self.manager)
if new_interface_hash == self.interface_hash:
self.manager.log("Cached module {} has same interface".format(self.id))
else:
self.manager.log("Cached module {} has changed interface".format(self.id))
self.mark_interface_stale()
self.interface_hash = new_interface_hash
if not self.path or self.options.cache_dir == os.devnull:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alright, we can still override writing the cache by setting /dev/null.

👍

return
if self.manager.options.quick_and_dirty:
is_errors = self.manager.errors.is_errors_for_file(self.path)
else:
is_errors = self.manager.errors.is_errors()
if is_errors:
return
dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies]
new_interface_hash = write_cache(
self.id, self.path, self.tree,
list(self.dependencies), list(self.suppressed), list(self.child_modules),
dep_prios, self.interface_hash,
self.manager)
if new_interface_hash == self.interface_hash:
self.manager.log("Cached module {} has same interface".format(self.id))
else:
self.manager.log("Cached module {} has changed interface".format(self.id))
self.mark_interface_stale()
self.interface_hash = new_interface_hash


def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
Expand Down