From 4e8d414dd14f7282ddbc85a9207dae06f68160d3 Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@jetbrains.com>
Date: Thu, 26 Jan 2017 18:51:13 +0300
Subject: [PATCH 01/10] Convention for test data which uses APIs defined by
 Python stubs

Currently we check only for syntax errors in stub files. The idea is to
add test data for static analyzers similar to how it's done in
DefinitelyTyped for TypeScript.
---
 CONTRIBUTING.md                            |  2 ++
 test_data/stdlib/2and3/collections_test.py | 14 ++++++++++++++
 tests/mypy_test.py                         |  2 +-
 tests/pytype_test.py                       |  8 +++++++-
 4 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 test_data/stdlib/2and3/collections_test.py

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index bf1eaa689eb7..616538dab9e6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -14,6 +14,8 @@ are important to the project's success.
     * [Contact us](#discussion) before starting significant work.
     * IMPORTANT: For new libraries, [get permission from the library owner first](#adding-a-new-library).
     * Create your stubs [conforming to the coding style](#stub-file-coding-style).
+    * Add tests for your stubs to `test_data`. These tests are not meant
+      to be executable. Type checkers analyze them statically.
     * Make sure `runtests.sh` passes cleanly on Mypy, pytype, and flake8.
 4. [Submit your changes](#submitting-changes):
     * Open a pull request
diff --git a/test_data/stdlib/2and3/collections_test.py b/test_data/stdlib/2and3/collections_test.py
new file mode 100644
index 000000000000..114bac7ad1df
--- /dev/null
+++ b/test_data/stdlib/2and3/collections_test.py
@@ -0,0 +1,14 @@
+from collections import namedtuple
+
+
+def test_collections_namedtuple():
+    # type: () -> None
+    Point = namedtuple('Point', 'x y')
+    p = Point(1, 2)
+
+    p._replace(y=3.14)
+    p._asdict()
+    p.x, p.y
+    p[0] + p[1]
+    p.index(1)
+    Point._make([('x', 1), ('y', 3.14)])
diff --git a/tests/mypy_test.py b/tests/mypy_test.py
index d474009e4d01..6a4e53510e60 100755
--- a/tests/mypy_test.py
+++ b/tests/mypy_test.py
@@ -63,7 +63,7 @@ def libpath(major, minor):
     versions.append('2and3')
     paths = []
     for v in versions:
-        for top in ['stdlib', 'third_party']:
+        for top in ['stdlib', 'third_party', 'test_data/stdlib', 'test_data/third_party']:
             p = os.path.join(top, v)
             if os.path.isdir(p):
                 paths.append(p)
diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index e7475c0be115..60cc4ae029f6 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -70,7 +70,7 @@ def pytype_test(args):
         print("Cannot run pytd. Did you install pytype?")
         return 0, 0
 
-    wanted = re.compile(r"stdlib/(2|2\.7|2and3)/.*\.pyi$")
+    wanted = re.compile(r"stdlib/(2|2\.7|2and3)/.*\.pyi?$")
     skipped = re.compile("(%s)$" % "|".join(load_blacklist()))
     files = []
 
@@ -80,6 +80,12 @@ def pytype_test(args):
             if wanted.search(f) and not skipped.search(f):
                 files.append(f)
 
+    for root, _, filenames in os.walk("test_data/stdlib"):
+        for f in sorted(filenames):
+            f = os.path.join(root, f)
+            if wanted.search(f) and not skipped.search(f):
+                files.append(f)
+
     running_tests = collections.deque()
     max_code, runs, errors = 0, 0, 0
     print("Running pytype tests...")

From 4066d7ff0672a3f350341019708bb280c05c5ff8 Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Sun, 5 Feb 2017 23:18:49 +0300
Subject: [PATCH 02/10] Run-time + static tests for stubs

---
 .travis.yml                                |  8 ++++--
 test_data/stdlib/2and3/collections_test.py | 17 +++++++------
 test_data/third_party/2and3/six_test.py    | 17 +++++++++++++
 tests/mypy_test.py                         |  1 +
 tests/runtime_test.py                      | 29 ++++++++++++++++++++++
 5 files changed, 62 insertions(+), 10 deletions(-)
 create mode 100644 test_data/third_party/2and3/six_test.py
 create mode 100644 tests/runtime_test.py

diff --git a/.travis.yml b/.travis.yml
index fd5905316981..b743514ef1c0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,12 +9,16 @@ matrix:
       env: TEST_CMD="./tests/mypy_test.py"
     - python: "2.7"
       env: TEST_CMD="./tests/pytype_test.py --num-parallel=4"
+    - python: "3.5"
+      env: TEST_CMD="./tests/runtime_test.py"
+    - python: "2.7"
+      env: TEST_CMD="./tests/runtime_test.py"
 
 install:
   # pytype needs py-2.7, mypy needs py-3.2+. Additional logic in runtests.py
   - if [[ $TRAVIS_PYTHON_VERSION == '3.6-dev' ]]; then pip install -U flake8==3.2.1 flake8-bugbear>=16.12.2 flake8-pyi>=16.12.2; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U git+git://github.com/python/mypy git+git://github.com/dropbox/typed_ast; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U git+git://github.com/google/pytype; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U pytest git+git://github.com/python/mypy git+git://github.com/dropbox/typed_ast; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U pytest git+git://github.com/google/pytype; fi
 
 script:
   - $TEST_CMD
diff --git a/test_data/stdlib/2and3/collections_test.py b/test_data/stdlib/2and3/collections_test.py
index 114bac7ad1df..2f1f71c35e55 100644
--- a/test_data/stdlib/2and3/collections_test.py
+++ b/test_data/stdlib/2and3/collections_test.py
@@ -1,14 +1,15 @@
 from collections import namedtuple
 
 
-def test_collections_namedtuple():
-    # type: () -> None
+def test_namedtuple():
     Point = namedtuple('Point', 'x y')
     p = Point(1, 2)
 
-    p._replace(y=3.14)
-    p._asdict()
-    p.x, p.y
-    p[0] + p[1]
-    p.index(1)
-    Point._make([('x', 1), ('y', 3.14)])
+    assert p == Point(1, 2)
+    assert p == (1, 2)
+    assert p._replace(y=3.14).y == 3.14
+    assert p._asdict()['x'] == 1
+    assert (p.x, p.y) == (1, 2)
+    assert p[0] + p[1] == 3
+    assert p.index(1) == 0
+    assert Point._make([1, 3.14]).y == 3.14
diff --git a/test_data/third_party/2and3/six_test.py b/test_data/third_party/2and3/six_test.py
new file mode 100644
index 000000000000..615fbcc7f396
--- /dev/null
+++ b/test_data/third_party/2and3/six_test.py
@@ -0,0 +1,17 @@
+import pip  # type: ignore
+
+pip.main('install six==1.10.0'.split())
+
+from six import PY2, PY3
+from six.moves import xrange
+
+
+def test_python_checks():
+    assert PY2 ^ PY3
+
+
+def test_xrange():
+    xs = xrange(4)
+    assert xs.__iter__
+    assert xs[0] == 0
+    assert sum(xs, 10)
diff --git a/tests/mypy_test.py b/tests/mypy_test.py
index 6a4e53510e60..af8ca2c57853 100755
--- a/tests/mypy_test.py
+++ b/tests/mypy_test.py
@@ -124,6 +124,7 @@ def main():
             runs += 1
             flags = ['--python-version', '%d.%d' % (major, minor)]
             flags.append('--strict-optional')
+            flags.append('--check-untyped-defs')
             if (major, minor) >= (3, 6):
                 flags.append('--fast-parser')
             # flags.append('--warn-unused-ignores')  # Fast parser and regular parser disagree.
diff --git a/tests/runtime_test.py b/tests/runtime_test.py
new file mode 100644
index 000000000000..7f89cfe814b2
--- /dev/null
+++ b/tests/runtime_test.py
@@ -0,0 +1,29 @@
+import pytest
+import sys
+import os
+
+
+def main():
+    if sys.version_info < (3, 0):
+        version_dirs = [
+            '2and3',
+            '2',
+        ]
+    else:
+        version_dirs = [
+            '2and3',
+            '3',
+            '%d.%d' % (sys.version_info[0], sys.version_info[1]),
+        ]
+    top_dirs = ['stdlib', 'third_party']
+    possible_paths = [os.path.join('test_data', t, v)
+                      for t in top_dirs
+                      for v in version_dirs]
+    print(possible_paths)
+    paths = [path for path in possible_paths if os.path.exists(path)]
+    print(paths)
+    pytest.main(paths)
+
+
+if __name__ == '__main__':
+    main()

From 4f2e158dedf1aa4ea029fa32283beefa1b2d4ce6 Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Sun, 5 Feb 2017 23:40:16 +0300
Subject: [PATCH 03/10] Mentioned both static and run-time checking in
 CONTRIBUTING

---
 CONTRIBUTING.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 616538dab9e6..cd826c3ff0e9 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -14,8 +14,9 @@ are important to the project's success.
     * [Contact us](#discussion) before starting significant work.
     * IMPORTANT: For new libraries, [get permission from the library owner first](#adding-a-new-library).
     * Create your stubs [conforming to the coding style](#stub-file-coding-style).
-    * Add tests for your stubs to `test_data`. These tests are not meant
-      to be executable. Type checkers analyze them statically.
+    * Add tests for your stubs to `test_data`. These tests should are
+      checked both statically via a type checker and at run-time via
+      pytest.
     * Make sure `runtests.sh` passes cleanly on Mypy, pytype, and flake8.
 4. [Submit your changes](#submitting-changes):
     * Open a pull request

From 992df42a5dfb96697c647506b5088eabcd69f50b Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Sun, 5 Feb 2017 23:40:44 +0300
Subject: [PATCH 04/10] Made runtime_test.py executable

---
 tests/runtime_test.py | 2 ++
 1 file changed, 2 insertions(+)
 mode change 100644 => 100755 tests/runtime_test.py

diff --git a/tests/runtime_test.py b/tests/runtime_test.py
old mode 100644
new mode 100755
index 7f89cfe814b2..0a42c15b46aa
--- a/tests/runtime_test.py
+++ b/tests/runtime_test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 import pytest
 import sys
 import os

From 774d5ffc0aadc25ad3eeb6bd70c9c22abe1c734a Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Sun, 5 Feb 2017 23:41:57 +0300
Subject: [PATCH 05/10] Ignore flake8 warnings about imports not at the top of
 the file for a while

---
 test_data/third_party/2and3/six_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test_data/third_party/2and3/six_test.py b/test_data/third_party/2and3/six_test.py
index 615fbcc7f396..1ac0351e04fc 100644
--- a/test_data/third_party/2and3/six_test.py
+++ b/test_data/third_party/2and3/six_test.py
@@ -2,8 +2,8 @@
 
 pip.main('install six==1.10.0'.split())
 
-from six import PY2, PY3
-from six.moves import xrange
+from six import PY2, PY3  # noqa: E402
+from six.moves import xrange  # noqa: E402
 
 
 def test_python_checks():

From ba61c115b7e7518c59c6e13287d2a4201d0f937b Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Sun, 5 Feb 2017 23:52:58 +0300
Subject: [PATCH 06/10] Removed debug output for checking Python version at
 Travis

---
 tests/runtime_test.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/runtime_test.py b/tests/runtime_test.py
index 0a42c15b46aa..5d5755908c18 100755
--- a/tests/runtime_test.py
+++ b/tests/runtime_test.py
@@ -21,9 +21,7 @@ def main():
     possible_paths = [os.path.join('test_data', t, v)
                       for t in top_dirs
                       for v in version_dirs]
-    print(possible_paths)
     paths = [path for path in possible_paths if os.path.exists(path)]
-    print(paths)
     pytest.main(paths)
 
 

From 41d5b7c3b56934c8f85fdcf3313a57ce2f5f04a9 Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Tue, 7 Feb 2017 03:27:19 +0300
Subject: [PATCH 07/10] Install *_requirements.txt for *_test.py automatically
 during test run

---
 test_data/conftest.py                            | 12 ++++++++++++
 test_data/pytest.ini                             |  2 ++
 test_data/stdlib/2and3/collections_test.py       |  5 ++---
 test_data/third_party/2and3/six_requirements.txt |  1 +
 test_data/third_party/2and3/six_test.py          | 16 ++++++----------
 5 files changed, 23 insertions(+), 13 deletions(-)
 create mode 100644 test_data/conftest.py
 create mode 100644 test_data/pytest.ini
 create mode 100644 test_data/third_party/2and3/six_requirements.txt

diff --git a/test_data/conftest.py b/test_data/conftest.py
new file mode 100644
index 000000000000..7bb182a18885
--- /dev/null
+++ b/test_data/conftest.py
@@ -0,0 +1,12 @@
+import re
+import pip
+import pytest
+
+
+@pytest.fixture(scope='module')
+def requirements(request):
+    requirements_path = re.sub(r'(.*)_test\.py', r'\1_requirements.txt',
+                               request.module.__file__)
+    pip.main(['install', '-r', requirements_path])
+    yield
+    # We could uninstall everything here after the module tests finish
diff --git a/test_data/pytest.ini b/test_data/pytest.ini
new file mode 100644
index 000000000000..33740040f9df
--- /dev/null
+++ b/test_data/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+usefixtures = requirements
diff --git a/test_data/stdlib/2and3/collections_test.py b/test_data/stdlib/2and3/collections_test.py
index 2f1f71c35e55..6ef1fd533607 100644
--- a/test_data/stdlib/2and3/collections_test.py
+++ b/test_data/stdlib/2and3/collections_test.py
@@ -1,7 +1,6 @@
-from collections import namedtuple
-
-
 def test_namedtuple():
+    from collections import namedtuple
+
     Point = namedtuple('Point', 'x y')
     p = Point(1, 2)
 
diff --git a/test_data/third_party/2and3/six_requirements.txt b/test_data/third_party/2and3/six_requirements.txt
new file mode 100644
index 000000000000..b6e34eb294e6
--- /dev/null
+++ b/test_data/third_party/2and3/six_requirements.txt
@@ -0,0 +1 @@
+six==1.10.0
diff --git a/test_data/third_party/2and3/six_test.py b/test_data/third_party/2and3/six_test.py
index 1ac0351e04fc..ab9898eb84b7 100644
--- a/test_data/third_party/2and3/six_test.py
+++ b/test_data/third_party/2and3/six_test.py
@@ -1,17 +1,13 @@
-import pip  # type: ignore
-
-pip.main('install six==1.10.0'.split())
-
-from six import PY2, PY3  # noqa: E402
-from six.moves import xrange  # noqa: E402
-
-
 def test_python_checks():
+    from six import PY2, PY3
+
     assert PY2 ^ PY3
 
 
 def test_xrange():
-    xs = xrange(4)
+    from six.moves import xrange
+
+    xs = xrange(5)
     assert xs.__iter__
     assert xs[0] == 0
-    assert sum(xs, 10)
+    assert sum(xs) == 10

From aa4ace141ce57ffe6072157af89af73285f4b493 Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Tue, 7 Feb 2017 04:10:55 +0300
Subject: [PATCH 08/10] Added a section about testing stubs with an example

---
 CONTRIBUTING.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 64 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cd826c3ff0e9..4105c0ca105c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -14,9 +14,7 @@ are important to the project's success.
     * [Contact us](#discussion) before starting significant work.
     * IMPORTANT: For new libraries, [get permission from the library owner first](#adding-a-new-library).
     * Create your stubs [conforming to the coding style](#stub-file-coding-style).
-    * Add tests for your stubs to `test_data`. These tests should are
-      checked both statically via a type checker and at run-time via
-      pytest.
+    * Optionally, [add some unit tests for your stubs](#adding-tests-for-stubs).
     * Make sure `runtests.sh` passes cleanly on Mypy, pytype, and flake8.
 4. [Submit your changes](#submitting-changes):
     * Open a pull request
@@ -216,6 +214,69 @@ documentation.  Whenever you find them disagreeing, model the type
 information after the actual implementation and file an issue on the
 project's tracker to fix their documentation.
 
+### Adding tests for stubs
+
+If you're fixing a stub file due to some false errors reported by your type
+checker, consider adding a unit test that would show that your changes actually
+fix these false errors.
+
+For example, Mypy is showing false errors about a hypothetical `example`
+module:
+
+```python
+import example
+
+example.foo()  # error: "module" has no attribute "foo"
+example.bar(10)  # error: Argument 1 to "bar" has incompatible type "int"
+```
+
+You might start with a pytest test that passes at run-time, but fails
+(incorrectly) during type checking. For a third-party library `example` that
+runs on both Python 2 and 3 put it into
+`test_data/third_party/2and3/example_test.py`:
+
+```python
+def test_foo_bar():
+    import example
+    
+    example.foo()
+    assert example.bar(10) - 10 == 0
+```
+
+Since `example` is a third-party library, create `example_requirements.txt`
+with the requirements specs for your test next to your test file:
+
+```python
+example==1.2.0
+```
+
+pytest will install the requirements automatically via a test fixture.
+
+You should put your third-party imports inside test functions in your test file,
+so the imports are not executed until pytype installs your requirements.
+
+Then, run both static and run-time tests for your test data. Alternatively, run
+them for all the test data:
+
+```python
+./tests/mypy_test.py
+./tests/runtime_test.py
+```
+
+The first one should fail with a false error (since you haven't fixed it yet)
+while the second one should pass, showing that you're using the API of `example`
+correctly.
+
+Next, add the stub file `third_party/2and3/example.pyi` that actually fixes
+these false errors:
+
+```python
+def foo() -> None: ...
+def bar(x: int) -> int: ...
+```
+
+Finally, re-run the tests to make sure the problem has gone.
+
 ## Issue-tracker conventions
 
 We aim to reply to all new issues promptly.  We'll assign a milestone

From dfca122cf5780148cb0eda5c58d7a6e45e165fb7 Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@gmail.com>
Date: Fri, 10 Feb 2017 03:45:51 +0300
Subject: [PATCH 09/10] Temporarily turned off running pytype for test_data

---
 tests/pytype_test.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 60cc4ae029f6..7a2c8b706fab 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -80,12 +80,6 @@ def pytype_test(args):
             if wanted.search(f) and not skipped.search(f):
                 files.append(f)
 
-    for root, _, filenames in os.walk("test_data/stdlib"):
-        for f in sorted(filenames):
-            f = os.path.join(root, f)
-            if wanted.search(f) and not skipped.search(f):
-                files.append(f)
-
     running_tests = collections.deque()
     max_code, runs, errors = 0, 0, 0
     print("Running pytype tests...")

From 3a3c6ad11f3f13ff33da8f12ad260c25b42184fd Mon Sep 17 00:00:00 2001
From: Andrey Vlasovskikh <andrey.vlasovskikh@jetbrains.com>
Date: Fri, 10 Feb 2017 19:17:02 +0300
Subject: [PATCH 10/10] Reverted running pytype_test (pytd, as it turned out)
 for *.py files

---
 tests/pytype_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/pytype_test.py b/tests/pytype_test.py
index 7a2c8b706fab..e7475c0be115 100755
--- a/tests/pytype_test.py
+++ b/tests/pytype_test.py
@@ -70,7 +70,7 @@ def pytype_test(args):
         print("Cannot run pytd. Did you install pytype?")
         return 0, 0
 
-    wanted = re.compile(r"stdlib/(2|2\.7|2and3)/.*\.pyi?$")
+    wanted = re.compile(r"stdlib/(2|2\.7|2and3)/.*\.pyi$")
     skipped = re.compile("(%s)$" % "|".join(load_blacklist()))
     files = []