Avoid candidates of the same version are created

Since 41a3008, Candidate objects prepare their underlying
distribution eagerly on creation, so the error can be caught
deterministically to trigger backtracking.

This however has a negative side-effect. Since dist preparation involves
metadata validation, a remote distribution must be downloaded on
Candidate creation. This means that an sdist will be downloaded,
validated, and discarded (since its version is known to be incompatible)
during backtracking.

This commit moves version deduplication of candidates from indexes to
before the Candidate object is created, to avoid unneeded preparation.

Note that we still need another round of deduplication in FoundCandidates
to remove duplicated candidates when a distribution is already installed.

Author: uranusjr

src/pip/_internal/resolution/resolvelib/factory.py

@@ -228,9 +228,13 @@ def iter_index_candidates():
             all_yanked = all(ican.link.is_yanked for ican in icans)
 
             # PackageFinder returns earlier versions first, so we reverse.
+            yielded = set()  # type: Set[_BaseVersion]
             for ican in reversed(icans):
                 if not all_yanked and ican.link.is_yanked:
                     continue
+                if ican.version in yielded:
+                    continue
+                yielded.add(ican.version)
                 candidate = self._make_candidate_from_link(
                     link=ican.link,
                     extras=extras,

src/pip/_internal/resolution/resolvelib/found_candidates.py

@@ -16,23 +16,11 @@
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
-    from typing import Callable, Iterator, Optional, Set
-
-    from pip._vendor.packaging.version import _BaseVersion
+    from typing import Callable, Iterator, Optional
 
     from .base import Candidate
 
 
-def _deduplicated_by_version(candidates):
-    # type: (Iterator[Candidate]) -> Iterator[Candidate]
-    returned = set()  # type: Set[_BaseVersion]
-    for candidate in candidates:
-        if candidate.version in returned:
-            continue
-        returned.add(candidate.version)
-        yield candidate
-
-
 def _insert_installed(installed, others):
     # type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate]
     """Iterator for ``FoundCandidates``.
@@ -86,12 +74,15 @@ def __getitem__(self, index):
     def __iter__(self):
         # type: () -> Iterator[Candidate]
         if not self._installed:
-            candidates = self._get_others()
-        elif self._prefers_installed:
-            candidates = itertools.chain([self._installed], self._get_others())
-        else:
-            candidates = _insert_installed(self._installed, self._get_others())
-        return _deduplicated_by_version(candidates)
+            return self._get_others()
+        others = (
+            candidate
+            for candidate in self._get_others()
+            if candidate.version != self._installed.version
+        )
+        if self._prefers_installed:
+            return itertools.chain([self._installed], others)
+        return _insert_installed(self._installed, others)
 
     def __len__(self):
         # type: () -> int