Find unused stubtest whitelist entries

.github/workflows/stubtest-unused-whitelist.yml

@@ -0,0 +1,26 @@
+name: Find unused stubtest whitelist entries
+
+on:
+  schedule:
+    - cron: '0 4 * * *'
+
+jobs:
+  find:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "windows-latest"]
+        python-version: [3.5, 3.6, 3.7, 3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          pip install -U git+git://github.com/python/mypy@b3d43984
+      - name: Run stubtest
+        run: ./tests/stubtest_unused.py

.travis.yml

@@ -42,44 +42,44 @@ jobs:
     - name: "stubtest py38"
       python: 3.8
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py37"
       python: 3.7
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py36"
       python: 3.6
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py35"
       python: 3.5
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py38 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.8.3
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py37 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.7.7
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py36 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.6.8
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py35 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.5.4
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist

tests/stubtest_test.py

@@ -21,6 +21,8 @@ def run_stubtest(typeshed_dir: Path) -> int:
     platform_whitelist = "{}.txt".format(sys.platform)
     combined_whitelist = "{}-py{}{}.txt".format(sys.platform, sys.version_info.major, sys.version_info.minor)
 
+    ignore_unused_whitelist = "--ignore-unused-whitelist" in sys.argv[1:]
+
     cmd = [
         sys.executable,
         "-m",
@@ -37,6 +39,8 @@ def run_stubtest(typeshed_dir: Path) -> int:
         "--whitelist",
         str(whitelist_dir / version_whitelist),
     ]
+    if ignore_unused_whitelist:
+        cmd += ["--ignore-unused-whitelist"]
     if (whitelist_dir / platform_whitelist).exists():
         cmd += [
             "--whitelist",
@@ -59,9 +63,6 @@ def run_stubtest(typeshed_dir: Path) -> int:
             "\nNB: stubtest output depends on the Python version (and system) it is run with. "
             "See README.md for more details.\n"
             "NB: We only check positional-only arg accuracy for Python 3.9.\n"
-            "If stubtest is complaining about 'unused whitelist entry' after your fix, please "
-            "remove the entry from the whitelist file. Note you may have to do this for other "
-            "version-specific whitelists as well. Thanks for helping burn the backlog of errors!\n"
             "\nCommand run was: {}\n".format(" ".join(cmd)),
             file=sys.stderr,
         )

tests/stubtest_unused.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+# Runs stubtest and prints each unused whitelist entry with filename.
+
+from typing import List, Tuple
+import os.path
+import subprocess
+import sys
+
+_UNUSED_NOTE = "note: unused whitelist entry "
+_WHITELIST_PATH = os.path.join("tests", "stubtest_whitelists")
+
+
+def main() -> int:
+    unused = run_stubtest()
+    with_filenames = []
+    for uu in unused:
+        with_filenames.extend(unused_files(uu))
+    for file, uu in with_filenames:
+        print(file + ":" + uu)
+    return 1 if with_filenames else 0
+
+
+def run_stubtest() -> List[str]:
+    popen = subprocess.Popen(
+        ["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
+    )
+    assert popen.stdout is not None
+    unused = []
+    for line in popen.stdout:
+        if line.startswith(_UNUSED_NOTE):
+            unused.append(line[len(_UNUSED_NOTE):].strip())
+    popen.wait()
+    return unused
+
+
+def unused_files(unused: str) -> List[Tuple[str, str]]:
+    version = "py{}{}".format(sys.version_info[0], sys.version_info[1])
+    files = ["py3_common.txt", version + ".txt", sys.platform + ".txt", sys.platform + "-" + version + ".txt"]
+    found = []
+    for file in files:
+        path = os.path.join(_WHITELIST_PATH, file)
+        if find_unused_in_file(unused, path):
+            found.append((path, unused))
+    if not found:
+        raise ValueError("unused item {} not found in any whitelist file".format(unused))
+    return found
+
+
+def find_unused_in_file(unused: str, path: str) -> bool:
+    try:
+        with open(path) as f:
+            return any(line.strip().split(" ")[0] == unused for line in f)
+    except FileNotFoundError:
+        return False
+
+
+if __name__ == "__main__":
+    sys.exit(main())