From 5a45762cf614e5d38d57cdbbd88e38a24e0cb3a7 Mon Sep 17 00:00:00 2001 From: Max Moroz Date: Sat, 29 Apr 2017 00:51:01 -0700 Subject: [PATCH 1/3] Ignore errors when deleting files and folders on Windows --- misc/incremental_checker.py | 8 ++------ misc/perf_checker.py | 7 ++----- misc/touch_checker.py | 9 +++------ mypy/myunit/__init__.py | 5 ++++- mypy/test/data.py | 8 ++++---- mypy/test/testcheck.py | 6 ++---- mypy/test/testcmdline.py | 3 ++- mypy/test/testpythoneval.py | 4 ++-- mypy/test/teststubgen.py | 5 +++-- mypy/util.py | 21 +++++++++++++++++++++ 10 files changed, 45 insertions(+), 31 deletions(-) diff --git a/misc/incremental_checker.py b/misc/incremental_checker.py index 515e662f76c5..8432312a72bc 100755 --- a/misc/incremental_checker.py +++ b/misc/incremental_checker.py @@ -39,12 +39,13 @@ import json import os import random -import shutil import subprocess import sys import textwrap import time +from mypy.util import delete_folder + CACHE_PATH = ".incremental_checker_cache.json" MYPY_REPO_URL = "https://github.com/python/mypy.git" @@ -59,11 +60,6 @@ def print_offset(text: str, indent_length: int = 4) -> None: print() -def delete_folder(folder_path: str) -> None: - if os.path.exists(folder_path): - shutil.rmtree(folder_path) - - def execute(command: List[str], fail_on_error: bool = True) -> Tuple[str, str, int]: proc = subprocess.Popen( ' '.join(command), diff --git a/misc/perf_checker.py b/misc/perf_checker.py index e55f8ccd38fe..d7c2b2539d75 100644 --- a/misc/perf_checker.py +++ b/misc/perf_checker.py @@ -9,6 +9,8 @@ import textwrap import time +from mypy.util import delete_folder + class Command: def __init__(self, setup: Callable[[], None], command: Callable[[], None]) -> None: @@ -22,11 +24,6 @@ def print_offset(text: str, indent_length: int = 4) -> None: print() -def delete_folder(folder_path: str) -> None: - if os.path.exists(folder_path): - shutil.rmtree(folder_path) - - def execute(command: List[str]) -> None: proc = subprocess.Popen( ' '.join(command), diff --git a/misc/touch_checker.py b/misc/touch_checker.py index c44afe492255..25a2bc538a42 100644 --- a/misc/touch_checker.py +++ b/misc/touch_checker.py @@ -11,6 +11,8 @@ import textwrap import time +from mypy.util import delete_folder + def print_offset(text: str, indent_length: int = 4) -> None: print() @@ -18,11 +20,6 @@ def print_offset(text: str, indent_length: int = 4) -> None: print() -def delete_folder(folder_path: str) -> None: - if os.path.exists(folder_path): - shutil.rmtree(folder_path) - - def execute(command: List[str]) -> None: proc = subprocess.Popen( ' '.join(command), @@ -121,7 +118,7 @@ def main() -> None: deltas = [] for filename in glob.iglob("mypy/**/*.py", recursive=True): print("{} {}".format(verb, filename)) - + setup, teardown = make_wrappers(filename) delta = test( setup, diff --git a/mypy/myunit/__init__.py b/mypy/myunit/__init__.py index 26b9a4574661..495c222e9563 100644 --- a/mypy/myunit/__init__.py +++ b/mypy/myunit/__init__.py @@ -131,7 +131,10 @@ def tear_down(self) -> None: if self.suite: self.suite.tear_down() os.chdir(self.old_cwd) - self.tmpdir.cleanup() + try: + self.tmpdir.cleanup() + except OSError: + pass self.old_cwd = None self.tmpdir = None diff --git a/mypy/test/data.py b/mypy/test/data.py index a0573d55479c..5913f7e813be 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -5,12 +5,12 @@ import posixpath import re from os import remove, rmdir -import shutil import pytest # type: ignore # no pytest in typeshed from typing import Callable, List, Tuple, Set, Optional, Iterator, Any from mypy.myunit import TestCase, SkipTestCaseException +from mypy.util import delete_folder, delete_file def parse_test_cases( @@ -228,13 +228,13 @@ def tear_down(self) -> None: # First remove files. for is_dir, path in reversed(self.clean_up): if not is_dir: - remove(path) + delete_file(path) # Then remove directories. for is_dir, path in reversed(self.clean_up): if is_dir: pycache = os.path.join(path, '__pycache__') if os.path.isdir(pycache): - shutil.rmtree(pycache) + delete_folder(pycache) try: rmdir(path) except OSError as error: @@ -251,7 +251,7 @@ def tear_down(self) -> None: # Be defensive -- only call rmtree if we're sure we aren't removing anything # valuable. if path.startswith('tmp/') and os.path.isdir(path): - shutil.rmtree(path) + delete_folder(path) raise super().tear_down() diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 7f95873bbd58..1db268b45f11 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -21,6 +21,7 @@ ) from mypy.errors import CompileError from mypy.options import Options +from mypy.util import delete_folder from mypy import experiments @@ -113,10 +114,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: experiments.STRICT_OPTIONAL = old_strict_optional def clear_cache(self) -> None: - dn = defaults.CACHE_DIR - - if os.path.exists(dn): - shutil.rmtree(dn) + delete_folder(defaults.CACHE_DIR) def run_case_once(self, testcase: DataDrivenTestCase, incremental: int = 0) -> None: find_module_clear_caches() diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 8e56b42bc766..6ecff97b6b68 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -17,6 +17,7 @@ from mypy.test.data import parse_test_cases, DataDrivenTestCase from mypy.test.helpers import assert_string_arrays_equal from mypy.version import __version__, base_version +from mypy.util import delete_file # Path to Python 3 interpreter python3_path = sys.executable @@ -58,7 +59,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] # Remove temp file. - os.remove(program_path) + delete_file(program_path) # Compare actual output to expected. if testcase.output_files: for path, expected_content in testcase.output_files: diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 85e9aa3751d7..a2e0b1de0b91 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -25,7 +25,7 @@ from mypy.test.config import test_data_prefix, test_temp_dir from mypy.test.data import DataDrivenTestCase, parse_test_cases from mypy.test.helpers import assert_string_arrays_equal -from mypy.util import try_find_python2_interpreter +from mypy.util import try_find_python2_interpreter, delete_file # Files which contain test case descriptions. @@ -94,7 +94,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: returncode, interp_out = run([interpreter, program], env=env) out += interp_out # Remove temp file. - os.remove(program_path) + delete_file(program_path) assert_string_arrays_equal(adapt_output(testcase), out, 'Invalid output ({}, line {})'.format( testcase.file, testcase.line)) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index aa76f2fbc90f..4c93e55f1799 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -22,6 +22,7 @@ parse_signature, parse_all_signatures, build_signature, find_unique_signatures, infer_sig_from_docstring ) +from mypy.util import delete_folder, delete_file class StubgenUtilSuite(Suite): @@ -136,8 +137,8 @@ def test_stubgen(testcase: DataDrivenTestCase) -> None: testcase.file, testcase.line)) finally: handle.close() - os.unlink(handle.name) - shutil.rmtree(out_dir) + delete_file(handle.name) + delete_folder(out_dir) def reset_importlib_caches() -> None: diff --git a/mypy/util.py b/mypy/util.py index 1e8e31898d23..9b4b20a0c153 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -2,6 +2,9 @@ import re import subprocess +import os +import sys +import shutil from xml.sax.saxutils import escape from typing import TypeVar, List, Tuple, Optional, Sequence, Dict @@ -134,3 +137,21 @@ def id(self, o: object) -> int: self.id_map[o] = self.next_id self.next_id += 1 return self.id_map[o] + + +def delete_folder(folder_path: str) -> None: + if sys.platform.startswith('win'): + shutil.rmtree(folder_path, ignore_errors=True) + else: + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + + +def delete_file(file_path: str) -> None: + if sys.platform.startswith('win'): + try: + os.remove(file_path) + except OSError: + pass + else: + os.remove(file_path) From 2860c5fb78c45e0437e86edefe350dbce1def7a7 Mon Sep 17 00:00:00 2001 From: Max Moroz Date: Sun, 30 Apr 2017 12:30:23 -0700 Subject: [PATCH 2/3] Remove dependency of misc on mypy --- misc/incremental_checker.py | 2 +- misc/perf_checker.py | 2 +- misc/touch_checker.py | 2 +- misc/util.py | 9 +++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 misc/util.py diff --git a/misc/incremental_checker.py b/misc/incremental_checker.py index 8432312a72bc..8b86c049b4c5 100755 --- a/misc/incremental_checker.py +++ b/misc/incremental_checker.py @@ -44,7 +44,7 @@ import textwrap import time -from mypy.util import delete_folder +from util import delete_folder CACHE_PATH = ".incremental_checker_cache.json" diff --git a/misc/perf_checker.py b/misc/perf_checker.py index d7c2b2539d75..8c33ecdb8285 100644 --- a/misc/perf_checker.py +++ b/misc/perf_checker.py @@ -9,7 +9,7 @@ import textwrap import time -from mypy.util import delete_folder +from util import delete_folder class Command: diff --git a/misc/touch_checker.py b/misc/touch_checker.py index 25a2bc538a42..4da6a7f0685f 100644 --- a/misc/touch_checker.py +++ b/misc/touch_checker.py @@ -11,7 +11,7 @@ import textwrap import time -from mypy.util import delete_folder +from util import delete_folder def print_offset(text: str, indent_length: int = 4) -> None: diff --git a/misc/util.py b/misc/util.py new file mode 100644 index 000000000000..7f1f3c879b97 --- /dev/null +++ b/misc/util.py @@ -0,0 +1,9 @@ +import os +import sys + +def delete_folder(folder_path: str) -> None: + if sys.platform.startswith('win'): + shutil.rmtree(folder_path, ignore_errors=True) + else: + if os.path.exists(folder_path): + shutil.rmtree(folder_path) From e293f94727544b279242b6aec2e9abc8908ee9f6 Mon Sep 17 00:00:00 2001 From: Max Moroz Date: Sun, 30 Apr 2017 14:37:42 -0700 Subject: [PATCH 3/3] Leave misc/ as is --- misc/incremental_checker.py | 8 ++++++-- misc/perf_checker.py | 7 +++++-- misc/touch_checker.py | 9 ++++++--- misc/util.py | 9 --------- 4 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 misc/util.py diff --git a/misc/incremental_checker.py b/misc/incremental_checker.py index 8b86c049b4c5..515e662f76c5 100755 --- a/misc/incremental_checker.py +++ b/misc/incremental_checker.py @@ -39,13 +39,12 @@ import json import os import random +import shutil import subprocess import sys import textwrap import time -from util import delete_folder - CACHE_PATH = ".incremental_checker_cache.json" MYPY_REPO_URL = "https://github.com/python/mypy.git" @@ -60,6 +59,11 @@ def print_offset(text: str, indent_length: int = 4) -> None: print() +def delete_folder(folder_path: str) -> None: + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + + def execute(command: List[str], fail_on_error: bool = True) -> Tuple[str, str, int]: proc = subprocess.Popen( ' '.join(command), diff --git a/misc/perf_checker.py b/misc/perf_checker.py index 8c33ecdb8285..e55f8ccd38fe 100644 --- a/misc/perf_checker.py +++ b/misc/perf_checker.py @@ -9,8 +9,6 @@ import textwrap import time -from util import delete_folder - class Command: def __init__(self, setup: Callable[[], None], command: Callable[[], None]) -> None: @@ -24,6 +22,11 @@ def print_offset(text: str, indent_length: int = 4) -> None: print() +def delete_folder(folder_path: str) -> None: + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + + def execute(command: List[str]) -> None: proc = subprocess.Popen( ' '.join(command), diff --git a/misc/touch_checker.py b/misc/touch_checker.py index 4da6a7f0685f..c44afe492255 100644 --- a/misc/touch_checker.py +++ b/misc/touch_checker.py @@ -11,8 +11,6 @@ import textwrap import time -from util import delete_folder - def print_offset(text: str, indent_length: int = 4) -> None: print() @@ -20,6 +18,11 @@ def print_offset(text: str, indent_length: int = 4) -> None: print() +def delete_folder(folder_path: str) -> None: + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + + def execute(command: List[str]) -> None: proc = subprocess.Popen( ' '.join(command), @@ -118,7 +121,7 @@ def main() -> None: deltas = [] for filename in glob.iglob("mypy/**/*.py", recursive=True): print("{} {}".format(verb, filename)) - + setup, teardown = make_wrappers(filename) delta = test( setup, diff --git a/misc/util.py b/misc/util.py deleted file mode 100644 index 7f1f3c879b97..000000000000 --- a/misc/util.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -import sys - -def delete_folder(folder_path: str) -> None: - if sys.platform.startswith('win'): - shutil.rmtree(folder_path, ignore_errors=True) - else: - if os.path.exists(folder_path): - shutil.rmtree(folder_path)