diff --git a/mypy/myunit/__init__.py b/mypy/myunit/__init__.py
index 92ba802530eb..8361f44b35b1 100644
--- a/mypy/myunit/__init__.py
+++ b/mypy/myunit/__init__.py
@@ -1,13 +1,12 @@
 import importlib
 import os
 import sys
-import re
 import tempfile
 import time
 import traceback
 
-from typing import List, Tuple, Any, Callable, Union, cast, Optional
-from types import TracebackType
+from typing import List, Tuple, Any, Callable, Union, cast, Optional, Iterable
+from types import TracebackType, MethodType
 
 
 # TODO remove global state
@@ -105,30 +104,23 @@ def fail() -> None:
     raise AssertionFailure()
 
 
-class TestCase:
-    def __init__(self, name: str, suite: 'Optional[Suite]' = None,
-                 func: Optional[Callable[[], None]] = None) -> None:
-        self.func = func
+class BaseTestCase:
+    """Common base class for _MyUnitTestCase and DataDrivenTestCase.
+
+    Handles temporary folder creation and deletion.
+    """
+    def __init__(self, name: str) -> None:
         self.name = name
-        self.suite = suite
         self.old_cwd = None  # type: Optional[str]
         self.tmpdir = None  # type: Optional[tempfile.TemporaryDirectory[str]]
 
-    def run(self) -> None:
-        if self.func:
-            self.func()
-
-    def set_up(self) -> None:
+    def setup(self) -> None:
         self.old_cwd = os.getcwd()
         self.tmpdir = tempfile.TemporaryDirectory(prefix='mypy-test-')
         os.chdir(self.tmpdir.name)
         os.mkdir('tmp')
-        if self.suite:
-            self.suite.set_up()
 
-    def tear_down(self) -> None:
-        if self.suite:
-            self.suite.tear_down()
+    def teardown(self) -> None:
         assert self.old_cwd is not None and self.tmpdir is not None, \
             "test was not properly set up"
         os.chdir(self.old_cwd)
@@ -140,35 +132,51 @@ def tear_down(self) -> None:
         self.tmpdir = None
 
 
+class _MyUnitTestCase(BaseTestCase):
+    """A concrete, myunit-specific test case, a wrapper around a method to run."""
+
+    def __init__(self, name: str, suite: 'Suite', run: Callable[[], None]) -> None:
+        super().__init__(name)
+        self.run = run
+        self.suite = suite
+
+    def setup(self) -> None:
+        super().setup()
+        self.suite.setup()
+
+    def teardown(self) -> None:
+        self.suite.teardown()  # No-op
+        super().teardown()
+
+
 class Suite:
-    def __init__(self) -> None:
-        self.prefix = typename(type(self)) + '.'
-        # Each test case is either a TestCase object or (str, function).
-        self._test_cases = []  # type: List[Any]
-        self.init()
+    """Abstract class for myunit test suites - node in the tree whose leaves are _MyUnitTestCases.
 
-    def set_up(self) -> None:
-        pass
+    The children `cases` are looked up during __init__, looking for attributes named test_*
+    they are either no-arg methods or of a pair (name, Suite).
+    """
 
-    def tear_down(self) -> None:
-        pass
+    cases = None  # type: Iterable[Union[_MyUnitTestCase, Tuple[str, Suite]]]
 
-    def init(self) -> None:
+    def __init__(self) -> None:
+        self.prefix = typename(type(self)) + '.'
+        self.cases = []
         for m in dir(self):
-            if m.startswith('test'):
+            if m.startswith('test_'):
                 t = getattr(self, m)
                 if isinstance(t, Suite):
-                    self.add_test((m + '.', t))
+                    self.cases.append((m + '.', t))
                 else:
-                    self.add_test(TestCase(m, self, getattr(self, m)))
+                    assert isinstance(t, MethodType)
+                    self.cases.append(_MyUnitTestCase(m, self, t))
 
-    def add_test(self, test: Union[TestCase,
-                                   Tuple[str, Callable[[], None]],
-                                   Tuple[str, 'Suite']]) -> None:
-        self._test_cases.append(test)
+    def setup(self) -> None:
+        """Set up fixtures"""
+        pass
 
-    def cases(self) -> List[Any]:
-        return self._test_cases[:]
+    def teardown(self) -> None:
+        # This method is not overridden in practice
+        pass
 
     def skip(self) -> None:
         raise SkipTestCaseException()
@@ -250,10 +258,11 @@ def main(args: Optional[List[str]] = None) -> None:
         sys.exit(1)
 
 
-def run_test_recursive(test: Any, num_total: int, num_fail: int, num_skip: int,
+def run_test_recursive(test: Union[_MyUnitTestCase, Tuple[str, Suite], ListSuite],
+                       num_total: int, num_fail: int, num_skip: int,
                        prefix: str, depth: int) -> Tuple[int, int, int]:
-    """The first argument may be TestCase, Suite or (str, Suite)."""
-    if isinstance(test, TestCase):
+    """The first argument may be _MyUnitTestCase, Suite or (str, Suite)."""
+    if isinstance(test, _MyUnitTestCase):
         name = prefix + test.name
         for pattern in patterns:
             if match_pattern(name, pattern):
@@ -275,7 +284,7 @@ def run_test_recursive(test: Any, num_total: int, num_fail: int, num_skip: int,
             suite = test
             suite_prefix = test.prefix
 
-        for stest in suite.cases():
+        for stest in suite.cases:
             new_prefix = prefix
             if depth > 0:
                 new_prefix = prefix + suite_prefix
@@ -284,14 +293,14 @@ def run_test_recursive(test: Any, num_total: int, num_fail: int, num_skip: int,
     return num_total, num_fail, num_skip
 
 
-def run_single_test(name: str, test: Any) -> Tuple[bool, bool]:
+def run_single_test(name: str, test: _MyUnitTestCase) -> Tuple[bool, bool]:
     if is_verbose:
         sys.stderr.write(name)
         sys.stderr.flush()
 
     time0 = time.time()
-    test.set_up()  # FIX: check exceptions
-    exc_traceback = None  # type: Any
+    test.setup()  # FIX: check exceptions
+    exc_traceback = None  # type: Optional[TracebackType]
     try:
         test.run()
     except BaseException as e:
@@ -299,7 +308,7 @@ def run_single_test(name: str, test: Any) -> Tuple[bool, bool]:
             raise
         exc_type, exc_value, exc_traceback = sys.exc_info()
     finally:
-        test.tear_down()
+        test.teardown()
     times.append((time.time() - time0, name))
 
     if exc_traceback:
diff --git a/mypy/test/config.py b/mypy/test/config.py
index 681f86664d4a..5dbe791e593a 100644
--- a/mypy/test/config.py
+++ b/mypy/test/config.py
@@ -1,9 +1,5 @@
-import os
 import os.path
 
-import typing
-
-
 this_file_dir = os.path.dirname(os.path.realpath(__file__))
 PREFIX = os.path.dirname(os.path.dirname(this_file_dir))
 
diff --git a/mypy/test/data.py b/mypy/test/data.py
index 189c71f2023c..509de8170947 100644
--- a/mypy/test/data.py
+++ b/mypy/test/data.py
@@ -6,13 +6,13 @@
 import re
 from os import remove, rmdir
 import shutil
+from abc import abstractmethod
 
 import pytest  # type: ignore  # no pytest in typeshed
-from typing import Callable, List, Tuple, Set, Optional, Iterator, Any, Dict, NamedTuple, Union
-
-from mypy.myunit import TestCase, SkipTestCaseException
-from mypy.test.config import test_temp_dir
+from typing import List, Tuple, Set, Optional, Iterator, Any, Dict, NamedTuple, Union
 
+from mypy.myunit import BaseTestCase
+from mypy.test.config import test_data_prefix, test_temp_dir
 
 root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
 
@@ -30,16 +30,14 @@
 
 def parse_test_cases(
         path: str,
-        perform: Optional[Callable[['DataDrivenTestCase'], None]],
         base_path: str = '.',
         optional_out: bool = False,
-        include_path: Optional[str] = None,
         native_sep: bool = False) -> List['DataDrivenTestCase']:
     """Parse a file with test case descriptions.
 
     Return an array of test cases.
 
-    NB this function and DataDrivenTestCase are shared between the
+    NB: this function and DataDrivenTestCase were shared between the
     myunit and pytest codepaths -- if something looks redundant,
     that's likely the reason.
     """
@@ -47,8 +45,7 @@ def parse_test_cases(
         join = os.path.join
     else:
         join = posixpath.join  # type: ignore
-    if not include_path:
-        include_path = os.path.dirname(path)
+    include_path = os.path.dirname(path)
     with open(path, encoding='utf-8') as f:
         lst = f.readlines()
     for i in range(len(lst)):
@@ -179,7 +176,7 @@ def parse_test_cases(
                 arg0 = p[i0].arg
                 assert arg0 is not None
                 tc = DataDrivenTestCase(arg0, input, tcout, tcout2, path,
-                                        p[i0].line, lastline, perform,
+                                        p[i0].line, lastline,
                                         files, output_files, stale_modules,
                                         rechecked_modules, deleted_paths, native_sep)
                 out.append(tc)
@@ -191,7 +188,12 @@ def parse_test_cases(
     return out
 
 
-class DataDrivenTestCase(TestCase):
+class DataDrivenTestCase(BaseTestCase):
+    """Holds parsed data and handles directory setup and teardown for MypyDataCase."""
+
+    # TODO: rename to ParsedTestCase or merge with MypyDataCase (yet avoid multiple inheritance)
+    # TODO: only create files on setup, not during parsing
+
     input = None  # type: List[str]
     output = None  # type: List[str]  # Output for the first pass
     output2 = None  # type: Dict[int, List[str]]  # Output for runs 2+, indexed by run number
@@ -215,7 +217,6 @@ def __init__(self,
                  file: str,
                  line: int,
                  lastline: int,
-                 perform: Optional[Callable[['DataDrivenTestCase'], None]],
                  files: List[Tuple[str, str]],
                  output_files: List[Tuple[str, str]],
                  expected_stale_modules: Dict[int, Set[str]],
@@ -230,7 +231,6 @@ def __init__(self,
         self.lastline = lastline
         self.file = file
         self.line = line
-        self.perform = perform
         self.files = files
         self.output_files = output_files
         self.expected_stale_modules = expected_stale_modules
@@ -238,8 +238,8 @@ def __init__(self,
         self.deleted_paths = deleted_paths
         self.native_sep = native_sep
 
-    def set_up(self) -> None:
-        super().set_up()
+    def setup(self) -> None:
+        super().setup()
         encountered_files = set()
         self.clean_up = []
         for paths in self.deleted_paths.values():
@@ -281,14 +281,7 @@ def add_dirs(self, dir: str) -> List[str]:
             os.mkdir(dir)
             return dirs
 
-    def run(self) -> None:
-        if self.name.endswith('-skip'):
-            raise SkipTestCaseException()
-        else:
-            assert self.perform is not None, 'Tests without `perform` should not be `run`'
-            self.perform(self)
-
-    def tear_down(self) -> None:
+    def teardown(self) -> None:
         # First remove files.
         for is_dir, path in reversed(self.clean_up):
             if not is_dir:
@@ -322,7 +315,7 @@ def tear_down(self) -> None:
                     if path.startswith(test_temp_dir + '/') and os.path.isdir(path):
                         shutil.rmtree(path)
                     raise
-        super().tear_down()
+        super().teardown()
 
     def find_steps(self) -> List[List[FileOperation]]:
         """Return a list of descriptions of file operations for each incremental step.
@@ -558,42 +551,74 @@ def pytest_addoption(parser: Any) -> None:
 
 # This function name is special to pytest.  See
 # http://doc.pytest.org/en/latest/writing_plugins.html#collection-hooks
-def pytest_pycollect_makeitem(collector: Any, name: str, obj: Any) -> Any:
-    if not isinstance(obj, type) or not issubclass(obj, DataSuite):
-        return None
-    return MypyDataSuite(name, parent=collector)
+def pytest_pycollect_makeitem(collector: Any, name: str,
+                              obj: object) -> 'Optional[Any]':
+    """Called by pytest on each object in modules configured in conftest.py files.
+
+    collector is pytest.Collector, returns Optional[pytest.Class]
+    """
+    if isinstance(obj, type):
+        # Only classes derived from DataSuite contain test cases, not the DataSuite class itself
+        if issubclass(obj, DataSuite) and obj is not DataSuite:
+            # Non-None result means this obj is a test case.
+            # The collect method of the returned MypyDataSuite instance will be called later,
+            # with self.obj being obj.
+            return MypyDataSuite(name, parent=collector)
+    return None
 
 
 class MypyDataSuite(pytest.Class):  # type: ignore  # inheriting from Any
-    def collect(self) -> Iterator['MypyDataCase']:
-        for case in self.obj.cases():
-            yield MypyDataCase(case.name, self, case)
+    def collect(self) -> Iterator[pytest.Item]:  # type: ignore
+        """Called by pytest on each of the object returned from pytest_pycollect_makeitem"""
+
+        # obj is the object for which pytest_pycollect_makeitem returned self.
+        suite = self.obj  # type: DataSuite
+        for f in suite.files:
+            for case in parse_test_cases(os.path.join(test_data_prefix, f),
+                                         base_path=suite.base_path,
+                                         optional_out=suite.optional_out,
+                                         native_sep=suite.native_sep):
+                if suite.filter(case):
+                    yield MypyDataCase(case.name, self, case)
+
+
+def is_incremental(testcase: DataDrivenTestCase) -> bool:
+    return 'incremental' in testcase.name.lower() or 'incremental' in testcase.file
+
+
+def has_stable_flags(testcase: DataDrivenTestCase) -> bool:
+    if any(re.match(r'# flags[2-9]:', line) for line in testcase.input):
+        return False
+    for filename, contents in testcase.files:
+        if os.path.basename(filename).startswith('mypy.ini.'):
+            return False
+    return True
 
 
 class MypyDataCase(pytest.Item):  # type: ignore  # inheriting from Any
-    def __init__(self, name: str, parent: MypyDataSuite, obj: DataDrivenTestCase) -> None:
+    def __init__(self, name: str, parent: MypyDataSuite, case: DataDrivenTestCase) -> None:
         self.skip = False
         if name.endswith('-skip'):
             self.skip = True
             name = name[:-len('-skip')]
 
         super().__init__(name, parent)
-        self.obj = obj
+        self.case = case
 
     def runtest(self) -> None:
         if self.skip:
             pytest.skip()
         update_data = self.config.getoption('--update-data', False)
-        self.parent.obj(update_data=update_data).run_case(self.obj)
+        self.parent.obj(update_data=update_data).run_case(self.case)
 
     def setup(self) -> None:
-        self.obj.set_up()
+        self.case.setup()
 
     def teardown(self) -> None:
-        self.obj.tear_down()
+        self.case.teardown()
 
     def reportinfo(self) -> Tuple[str, int, str]:
-        return self.obj.file, self.obj.line, self.obj.name
+        return self.case.file, self.case.line, self.case.name
 
     def repr_failure(self, excinfo: Any) -> str:
         if excinfo.errisinstance(SystemExit):
@@ -606,16 +631,23 @@ def repr_failure(self, excinfo: Any) -> str:
             self.parent._prunetraceback(excinfo)
             excrepr = excinfo.getrepr(style='short')
 
-        return "data: {}:{}:\n{}".format(self.obj.file, self.obj.line, excrepr)
+        return "data: {}:{}:\n{}".format(self.case.file, self.case.line, excrepr)
 
 
 class DataSuite:
+    # option fields - class variables
+    files = None  # type: List[str]
+    base_path = '.'  # type: str
+    optional_out = False  # type: bool
+    native_sep = False  # type: bool
+
     def __init__(self, *, update_data: bool) -> None:
         self.update_data = update_data
 
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        return []
-
+    @abstractmethod
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         raise NotImplementedError
+
+    @classmethod
+    def filter(cls, testcase: DataDrivenTestCase) -> bool:
+        return True
diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py
index 4e27e37a7e45..edee17b90ffb 100644
--- a/mypy/test/testargs.py
+++ b/mypy/test/testargs.py
@@ -5,7 +5,6 @@
 object it creates.
 """
 
-import typing
 from mypy.myunit import Suite, assert_equal
 from mypy.options import Options, BuildType
 from mypy.main import process_options
diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py
index 7a4a6f289a7b..b33dfba6405f 100644
--- a/mypy/test/testcheck.py
+++ b/mypy/test/testcheck.py
@@ -3,9 +3,6 @@
 import os
 import re
 import shutil
-import sys
-import time
-import typed_ast
 
 from typing import Dict, List, Optional, Set, Tuple
 
@@ -13,8 +10,8 @@
 from mypy.main import process_options
 from mypy.build import BuildSource, find_module_clear_caches
 from mypy.myunit import AssertionFailure
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages,
     retry_on_error, testcase_pyversion, update_testcase_output,
@@ -25,7 +22,7 @@
 from mypy import experiments
 
 # List of files that contain test case descriptions.
-files = [
+typecheck_files = [
     'check-basic.test',
     'check-callable.test',
     'check-classes.test',
@@ -83,14 +80,9 @@
 
 
 class TypeCheckSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
+    files = typecheck_files
+    base_path = test_temp_dir
+    optional_out = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         incremental = ('incremental' in testcase.name.lower()
diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py
index 26f1780559e1..afdee5583968 100644
--- a/mypy/test/testcmdline.py
+++ b/mypy/test/testcmdline.py
@@ -9,12 +9,12 @@
 import subprocess
 import sys
 
-from typing import Tuple, List, Dict, Set
+from typing import List
 
-from mypy.myunit import Suite, SkipTestCaseException, AssertionFailure
-from mypy.test.config import test_data_prefix, test_temp_dir
+from mypy.myunit import AssertionFailure
+from mypy.test.config import test_temp_dir
 from mypy.test.data import fix_cobertura_filename
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages
 from mypy.version import __version__, base_version
 
@@ -28,24 +28,17 @@
 ]
 
 
-class PythonEvaluationSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in cmdline_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_python_evaluation,
-                                  base_path=test_temp_dir,
-                                  optional_out=True,
-                                  native_sep=True)
-        return c
+class PythonCmdlineSuite(DataSuite):
+    files = cmdline_files
+    base_path = test_temp_dir
+    optional_out = True
+    native_sep = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
-        test_python_evaluation(testcase)
+        test_python_cmdline(testcase)
 
 
-def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
+def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
     assert testcase.old_cwd is not None, "test was not properly set up"
     # Write the program to a file.
     program = '_program.py'
diff --git a/mypy/test/testdeps.py b/mypy/test/testdeps.py
index 1cd1411d11ec..98ff104962eb 100644
--- a/mypy/test/testdeps.py
+++ b/mypy/test/testdeps.py
@@ -9,30 +9,23 @@
 from mypy.nodes import MypyFile, Expression
 from mypy.options import Options
 from mypy.server.deps import get_dependencies
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 from mypy.types import Type
 
-files = [
-    'deps.test',
-    'deps-types.test',
-    'deps-generics.test',
-    'deps-expressions.test',
-    'deps-statements.test',
-    'deps-classes.test',
-]
-
 
 class GetDependenciesSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
+    files = [
+        'deps.test',
+        'deps-types.test',
+        'deps-generics.test',
+        'deps-expressions.test',
+        'deps-statements.test',
+        'deps-classes.test',
+    ]
+    base_path = test_temp_dir
+    optional_out = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         src = '\n'.join(testcase.input)
diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py
index 248a4550b0c4..b1cfc65a4a29 100644
--- a/mypy/test/testdiff.py
+++ b/mypy/test/testdiff.py
@@ -9,25 +9,15 @@
 from mypy.nodes import MypyFile
 from mypy.options import Options
 from mypy.server.astdiff import snapshot_symbol_table, compare_symbol_table_snapshots
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 
 
-files = [
-    'diff.test'
-]
-
-
 class ASTDiffSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
+    files = ['diff.test']
+    base_path = test_temp_dir
+    optional_out = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         first_src = '\n'.join(testcase.input)
diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index 1f483e1bc315..82de0940f242 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -4,8 +4,6 @@
 import re
 import shutil
 import sys
-import time
-import typed_ast
 
 from typing import Dict, List, Optional, Set, Tuple
 
@@ -13,43 +11,40 @@
 from mypy import defaults
 from mypy.main import process_options
 from mypy.myunit import AssertionFailure
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite, has_stable_flags, is_incremental
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages,
     retry_on_error, testcase_pyversion, update_testcase_output,
 )
-from mypy.errors import CompileError
 from mypy.options import Options
 
-from mypy import experiments
 from mypy import dmypy
 
 # List of files that contain test case descriptions.
-files = [
-    'check-enum.test',
-    'check-incremental.test',
-    'check-newtype.test',
-]
+if sys.platform != 'win32':
+    dmypy_files = [
+        'check-enum.test',
+        'check-incremental.test',
+        'check-newtype.test',
+    ]
+else:
+    dmypy_files = []
 
 
 class TypeCheckSuite(DataSuite):
+    files = dmypy_files
+    base_path = test_temp_dir
+    optional_out = True
 
     @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        if sys.platform == 'win32':
-            return []  # Nothing here works on Windows.
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            tc = parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-            c += [case for case in tc
-                  if cls.has_stable_flags(case) and cls.is_incremental(case)]
-        return c
+    def filter(cls, testcase: DataDrivenTestCase) -> bool:
+        return has_stable_flags(testcase) and is_incremental(testcase)
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
-        assert self.is_incremental(testcase), "Testcase is not incremental"
-        assert self.has_stable_flags(testcase), "Testcase has varying flags"
+        assert has_stable_flags(testcase), "Testcase has varying flags"
+        assert is_incremental(testcase), "Testcase is not incremental"
+
         # All tests run once with a cold cache, then at least once
         # with a warm cache and maybe changed files.  Expected output
         # is specified separately for each run.
@@ -67,19 +62,6 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         for step in range(1, num_steps + 1):
             self.run_case_once(testcase, step)
 
-    @classmethod
-    def is_incremental(cls, testcase: DataDrivenTestCase) -> bool:
-        return 'incremental' in testcase.name.lower() or 'incremental' in testcase.file
-
-    @classmethod
-    def has_stable_flags(cls, testcase: DataDrivenTestCase) -> bool:
-        if any(re.match(r'# flags[2-9]:', line) for line in testcase.input):
-            return False
-        for filename, contents in testcase.files:
-            if os.path.basename(filename).startswith('mypy.ini.'):
-                return False
-        return True
-
     def clear_cache(self) -> None:
         dn = defaults.CACHE_DIR
         if os.path.exists(dn):
diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py
index 13a1a63f2585..b4379b25f63f 100644
--- a/mypy/test/testfinegrained.py
+++ b/mypy/test/testfinegrained.py
@@ -10,6 +10,7 @@
 import os
 import re
 import shutil
+
 from typing import List, Tuple, Dict, Optional, Set
 
 from mypy import build
@@ -29,22 +30,15 @@
 from mypy.util import short_type
 
 
-files = [
-    'fine-grained.test',
-    'fine-grained-cycles.test',
-    'fine-grained-blockers.test',
-    'fine-grained-modules.test',
-]
-
-
 class FineGrainedSuite(DataSuite):
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
+    files = [
+        'fine-grained.test',
+        'fine-grained-cycles.test',
+        'fine-grained-blockers.test',
+        'fine-grained-modules.test',
+    ]
+    base_path = test_temp_dir
+    optional_out = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         main_src = '\n'.join(testcase.input)
diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py
index c367e823610f..07a26d9b9c99 100644
--- a/mypy/test/testmerge.py
+++ b/mypy/test/testmerge.py
@@ -15,19 +15,14 @@
 from mypy.server.subexpr import get_subexpressions
 from mypy.server.update import FineGrainedBuildManager
 from mypy.strconv import StrConv, indent
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages
 from mypy.test.testtypegen import ignore_node
 from mypy.types import TypeStrVisitor, Type
 from mypy.util import short_type, IdMapper
 
 
-files = [
-    'merge.test'
-]
-
-
 # Which data structures to dump in a test case?
 SYMTABLE = 'SYMTABLE'
 TYPEINFO = ' TYPEINFO'
@@ -39,6 +34,10 @@
 
 
 class ASTMergeSuite(DataSuite):
+    files = ['merge.test']
+    base_path = test_temp_dir
+    optional_out = True
+
     def __init__(self, *, update_data: bool) -> None:
         super().__init__(update_data=update_data)
         self.str_conv = StrConv(show_ids=True)
@@ -46,14 +45,6 @@ def __init__(self, *, update_data: bool) -> None:
         self.id_mapper = self.str_conv.id_mapper  # type: IdMapper
         self.type_str_conv = TypeStrVisitor(self.id_mapper)
 
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
-
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         name = testcase.name
         # We use the test case name to decide which data structures to dump.
diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py
index ef9632a34372..5646ed57c6c9 100644
--- a/mypy/test/testparse.py
+++ b/mypy/test/testparse.py
@@ -1,31 +1,18 @@
 """Tests for the mypy parser."""
-
-import os.path
-
 from typing import List
 
 from mypy import defaults
-from mypy.myunit import Suite, AssertionFailure
+from mypy.myunit import AssertionFailure
 from mypy.test.helpers import assert_string_arrays_equal
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
-from mypy.test import config
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.parse import parse
 from mypy.errors import CompileError
 from mypy.options import Options
 
 
 class ParserSuite(DataSuite):
-    parse_files = ['parse.test',
-                   'parse-python2.test']
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        # The test case descriptions are stored in data files.
-        c = []  # type: List[DataDrivenTestCase]
-        for f in cls.parse_files:
-            c += parse_test_cases(
-                os.path.join(config.test_data_prefix, f), test_parser)
-        return c
+    files = ['parse.test',
+             'parse-python2.test']
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_parser(testcase)
@@ -62,12 +49,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None:
 
 
 class ParseErrorSuite(DataSuite):
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        # Test case descriptions are in an external file.
-        return parse_test_cases(os.path.join(config.test_data_prefix,
-                                             'parse-errors.test'),
-                                test_parse_error)
+    files = ['parse-errors.test']
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_parse_error(testcase)
diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py
index be32252a21b5..222fa6ff32c2 100644
--- a/mypy/test/testpythoneval.py
+++ b/mypy/test/testpythoneval.py
@@ -19,35 +19,23 @@
 import pytest  # type: ignore  # no pytest in typeshed
 from typing import Dict, List, Tuple, Optional
 
-from mypy.test.config import test_data_prefix, test_temp_dir
-from mypy.test.data import DataDrivenTestCase, parse_test_cases, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 from mypy.util import try_find_python2_interpreter
 from mypy import api
 
-# Files which contain test case descriptions.
-python_eval_files = ['pythoneval.test',
-                     'python2eval.test']
-
-python_34_eval_files = ['pythoneval-asyncio.test']
-
 # Path to Python 3 interpreter
 python3_path = sys.executable
 program_re = re.compile(r'\b_program.py\b')
 
 
 class PythonEvaluationSuite(DataSuite):
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in python_eval_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_python_evaluation, test_temp_dir, True)
-        if sys.version_info.major == 3 and sys.version_info.minor >= 4:
-            for f in python_34_eval_files:
-                c += parse_test_cases(os.path.join(test_data_prefix, f),
-                    test_python_evaluation, test_temp_dir, True)
-        return c
+    files = ['pythoneval.test',
+             'python2eval.test',
+             'pythoneval-asyncio.test']
+    base_path = test_temp_dir
+    optional_out = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_python_evaluation(testcase)
diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py
index 8ea7c1d5cc9f..98f3ef64b26c 100644
--- a/mypy/test/testsemanal.py
+++ b/mypy/test/testsemanal.py
@@ -9,8 +9,8 @@
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages, testfile_pyversion,
 )
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
-from mypy.test.config import test_data_prefix, test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
 from mypy.errors import CompileError
 from mypy.nodes import TypeInfo
 from mypy.options import Options
@@ -42,16 +42,10 @@ def get_semanal_options() -> Options:
 
 
 class SemAnalSuite(DataSuite):
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in semanal_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_semanal,
-                                  base_path=test_temp_dir,
-                                  optional_out=True,
-                                  native_sep=True)
-        return c
+    files = semanal_files
+    base_path = test_temp_dir
+    optional_out = True
+    native_sep = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_semanal(testcase)
@@ -101,19 +95,10 @@ def test_semanal(testcase: DataDrivenTestCase) -> None:
 
 # Semantic analyzer error test cases
 
-# Paths to files containing test case descriptions.
-semanal_error_files = ['semanal-errors.test']
-
-
 class SemAnalErrorSuite(DataSuite):
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        # Read test cases from test case description files.
-        c = []  # type: List[DataDrivenTestCase]
-        for f in semanal_error_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_semanal_error, test_temp_dir, optional_out=True)
-        return c
+    files = ['semanal-errors.test']
+    base_path = test_temp_dir
+    optional_out = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_semanal_error(testcase)
@@ -140,18 +125,9 @@ def test_semanal_error(testcase: DataDrivenTestCase) -> None:
 
 # SymbolNode table export test cases
 
-# Test case descriptions
-semanal_symtable_files = ['semanal-symtable.test']
-
-
 class SemAnalSymtableSuite(DataSuite):
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in semanal_symtable_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir)
-        return c
+    files = ['semanal-symtable.test']
+    base_path = test_temp_dir
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         """Perform a test case."""
@@ -179,19 +155,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
 
 
 # Type info export test cases
-
-semanal_typeinfo_files = ['semanal-typeinfo.test']
-
-
 class SemAnalTypeInfoSuite(DataSuite):
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        """Test case descriptions"""
-        c = []  # type: List[DataDrivenTestCase]
-        for f in semanal_typeinfo_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir)
-        return c
+    files = ['semanal-typeinfo.test']
+    base_path = test_temp_dir
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         """Perform a test case."""
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 539080c387f9..e292fdd1690e 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -1,21 +1,17 @@
 import glob
 import importlib
 import os.path
-import random
 import shutil
 import sys
 import tempfile
-import time
 import re
 from types import ModuleType
 
 from typing import List, Tuple
 
-from mypy.myunit import Suite, AssertionFailure, assert_equal
+from mypy.myunit import Suite, assert_equal
 from mypy.test.helpers import assert_string_arrays_equal
-from mypy.test.data import DataSuite, parse_test_cases, DataDrivenTestCase
-from mypy.test import config
-from mypy.parse import parse
+from mypy.test.data import DataSuite, DataDrivenTestCase
 from mypy.errors import CompileError
 from mypy.stubgen import generate_stub, generate_stub_for_module, parse_options, Options
 from mypy.stubgenc import generate_c_type_stub, infer_method_sig
@@ -96,14 +92,7 @@ def test_infer_sig_from_docstring(self) -> None:
 
 
 class StubgenPythonSuite(DataSuite):
-    test_data_files = ['stubgen.test']
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for path in cls.test_data_files:
-            c += parse_test_cases(os.path.join(config.test_data_prefix, path), test_stubgen)
-        return c
+    files = ['stubgen.test']
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_stubgen(testcase)
diff --git a/mypy/test/testsubtypes.py b/mypy/test/testsubtypes.py
index 7a19c6c1325b..2e5d960809d7 100644
--- a/mypy/test/testsubtypes.py
+++ b/mypy/test/testsubtypes.py
@@ -6,7 +6,7 @@
 
 
 class SubtypingSuite(Suite):
-    def set_up(self) -> None:
+    def setup(self) -> None:
         self.fx = TypeFixture(INVARIANT)
         self.fx_contra = TypeFixture(CONTRAVARIANT)
         self.fx_co = TypeFixture(COVARIANT)
diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py
index 852b96721d40..5f693b5d239a 100644
--- a/mypy/test/testtransform.py
+++ b/mypy/test/testtransform.py
@@ -1,16 +1,15 @@
 """Identity AST transform test cases"""
 
 import os.path
-
-from typing import Dict, List
+from typing import List
 
 from mypy import build
 from mypy.build import BuildSource
 from mypy.test.helpers import (
     assert_string_arrays_equal, testfile_pyversion, normalize_error_messages
 )
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
-from mypy.test.config import test_data_prefix, test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
 from mypy.errors import CompileError
 from mypy.treetransform import TransformVisitor
 from mypy.types import Type
@@ -19,24 +18,16 @@
 
 class TransformSuite(DataSuite):
     # Reuse semantic analysis test cases.
-    transform_files = ['semanal-basic.test',
-                       'semanal-expressions.test',
-                       'semanal-classes.test',
-                       'semanal-types.test',
-                       'semanal-modules.test',
-                       'semanal-statements.test',
-                       'semanal-abstractclasses.test',
-                       'semanal-python2.test']
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in cls.transform_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_transform,
-                                  base_path=test_temp_dir,
-                                  native_sep=True)
-        return c
+    files = ['semanal-basic.test',
+             'semanal-expressions.test',
+             'semanal-classes.test',
+             'semanal-types.test',
+             'semanal-modules.test',
+             'semanal-statements.test',
+             'semanal-abstractclasses.test',
+             'semanal-python2.test']
+    base_path = test_temp_dir
+    native_sep = True
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_transform(testcase)
diff --git a/mypy/test/testtypegen.py b/mypy/test/testtypegen.py
index 7376d51913ef..f953b6eaf13b 100644
--- a/mypy/test/testtypegen.py
+++ b/mypy/test/testtypegen.py
@@ -1,14 +1,13 @@
 """Test cases for the type checker: exporting inferred types"""
 
-import os.path
 import re
 
 from typing import Set, List
 
 from mypy import build
 from mypy.build import BuildSource
-from mypy.test import config
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 from mypy.util import short_type
 from mypy.nodes import (
@@ -20,16 +19,8 @@
 
 
 class TypeExportSuite(DataSuite):
-    # List of files that contain test case descriptions.
     files = ['typexport-basic.test']
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in cls.files:
-            c += parse_test_cases(os.path.join(config.test_data_prefix, f),
-                                  None, config.test_temp_dir)
-        return c
+    base_path = test_temp_dir
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         try:
@@ -44,7 +35,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             options.show_traceback = True
             result = build.build(sources=[BuildSource('main', None, src)],
                                  options=options,
-                                 alt_lib_path=config.test_temp_dir)
+                                 alt_lib_path=test_temp_dir)
             a = result.errors
             map = result.types
             nodes = map.keys()
diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py
index 8b025d66b40c..c8d258fc1168 100644
--- a/mypy/test/testtypes.py
+++ b/mypy/test/testtypes.py
@@ -93,7 +93,7 @@ def test_generic_function_type(self) -> None:
 
 
 class TypeOpsSuite(Suite):
-    def set_up(self) -> None:
+    def setup(self) -> None:
         self.fx = TypeFixture(INVARIANT)
         self.fx_co = TypeFixture(COVARIANT)
         self.fx_contra = TypeFixture(CONTRAVARIANT)
@@ -358,7 +358,7 @@ def callable(self, vars: List[str], *a: Type) -> CallableType:
 
 
 class JoinSuite(Suite):
-    def set_up(self) -> None:
+    def setup(self) -> None:
         self.fx = TypeFixture()
 
     def test_trivial_cases(self) -> None:
@@ -628,7 +628,7 @@ def type_callable(self, *a: Type) -> CallableType:
 
 
 class MeetSuite(Suite):
-    def set_up(self) -> None:
+    def setup(self) -> None:
         self.fx = TypeFixture()
 
     def test_trivial_cases(self) -> None: