diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py
index 635b99cbc0ff..f2710432a532 100644
--- a/mypy/test/testpythoneval.py
+++ b/mypy/test/testpythoneval.py
@@ -10,8 +10,6 @@
       this suite would slow down the main suite too much.
 """
 
-from contextlib import contextmanager
-import errno
 import os
 import os.path
 import re
@@ -25,6 +23,7 @@
 from mypy.test.data import DataDrivenTestCase, parse_test_cases, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 from mypy.util import try_find_python2_interpreter
+from mypy import api
 
 # Files which contain test case descriptions.
 python_eval_files = ['pythoneval.test',
@@ -61,11 +60,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
     version.
     """
     assert testcase.old_cwd is not None, "test was not properly set up"
-    mypy_cmdline = [
-        python3_path,
-        os.path.join(testcase.old_cwd, 'scripts', 'mypy'),
-        '--show-traceback',
-    ]
+    mypy_cmdline = ['--show-traceback']
     py2 = testcase.name.lower().endswith('python2')
     if py2:
         mypy_cmdline.append('--py2')
@@ -80,21 +75,27 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
 
     # Write the program to a file.
     program = '_' + testcase.name + '.py'
-    mypy_cmdline.append(program)
     program_path = os.path.join(test_temp_dir, program)
+    mypy_cmdline.append(program_path)
     with open(program_path, 'w') as file:
         for s in testcase.input:
             file.write('{}\n'.format(s))
+    output = []
     # Type check the program.
-    # This uses the same PYTHONPATH as the current process.
-    returncode, out = run(mypy_cmdline)
+    out, err, returncode = api.run(mypy_cmdline)
+    # split lines, remove newlines, and remove directory of test case
+    for line in (out + err).splitlines():
+        if line.startswith(test_temp_dir + os.sep):
+            output.append(line[len(test_temp_dir + os.sep):].rstrip("\r\n"))
+        else:
+            output.append(line.rstrip("\r\n"))
     if returncode == 0:
         # Execute the program.
         returncode, interp_out = run([interpreter, program])
-        out += interp_out
+        output.extend(interp_out)
     # Remove temp file.
     os.remove(program_path)
-    assert_string_arrays_equal(adapt_output(testcase), out,
+    assert_string_arrays_equal(adapt_output(testcase), output,
                                'Invalid output ({}, line {})'.format(
                                    testcase.file, testcase.line))
 
@@ -115,7 +116,7 @@ def adapt_output(testcase: DataDrivenTestCase) -> List[str]:
 
 
 def run(
-    cmdline: List[str], *, env: Optional[Dict[str, str]] = None, timeout: int = 30
+    cmdline: List[str], *, env: Optional[Dict[str, str]] = None, timeout: int = 300
 ) -> Tuple[int, List[str]]:
     """A poor man's subprocess.run() for 3.3 and 3.4 compatibility."""
     process = subprocess.Popen(