handle metadata email parsing errors

Author: cosmicexplorer

src/pip/_internal/exceptions.py

@@ -253,6 +253,25 @@ def __str__(self) -> str:
         )
 
 
+class CacheMetadataError(PipError):
+    """Raised when de/serializing a requirement into the metadata cache."""
+
+    def __init__(
+        self,
+        req: "InstallRequirement",
+        reason: str,
+    ) -> None:
+        """
+        :param req: The requirement we attempted to cache.
+        :param reason: Context about the precise error that occurred.
+        """
+        self.req = req
+        self.reason = reason
+
+    def __str__(self) -> str:
+        return f"{self.reason} for {self.req} from {self.req.link}"
+
+
 class UserInstallationInvalid(InstallationError):
     """A --user install is requested on an environment without user site."""
 

src/pip/_internal/metadata/__init__.py

@@ -6,7 +6,14 @@
 
 from pip._internal.utils.misc import strtobool
 
-from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel
+from .base import (
+    BaseDistribution,
+    BaseEnvironment,
+    FilesystemWheel,
+    MemoryWheel,
+    Wheel,
+    serialize_metadata,
+)
 
 if TYPE_CHECKING:
     from typing import Literal, Protocol
@@ -23,6 +30,7 @@
     "get_environment",
     "get_wheel_distribution",
     "select_backend",
+    "serialize_metadata",
 ]
 
 

src/pip/_internal/metadata/base.py

@@ -1,6 +1,9 @@
 import csv
+import email.generator
 import email.message
+import email.policy
 import functools
+import io
 import json
 import logging
 import pathlib
@@ -97,6 +100,18 @@ def _convert_installed_files_path(
     return str(pathlib.Path(*info, *entry))
 
 
+def serialize_metadata(msg: email.message.Message) -> str:
+    """Write a dist's metadata to a string.
+
+    Calling ``str(dist.metadata)`` may raise an error by misinterpreting RST directives
+    as email headers. This method uses the more robust ``email.policy.EmailPolicy`` to
+    avoid those parsing errors."""
+    out = io.StringIO()
+    g = email.generator.Generator(out, policy=email.policy.EmailPolicy())
+    g.flatten(msg)
+    return out.getvalue()
+
+
 class RequiresEntry(NamedTuple):
     requirement: str
     extra: str

src/pip/_internal/operations/prepare.py

@@ -4,7 +4,6 @@
 # The following comment should be removed at some point in the future.
 # mypy: strict-optional=False
 
-import email.errors
 import gzip
 import json
 import mimetypes
@@ -20,6 +19,7 @@
 from pip._internal.cache import LinkMetadataCache, should_cache
 from pip._internal.distributions import make_distribution_for_install_requirement
 from pip._internal.exceptions import (
+    CacheMetadataError,
     DirectoryUrlHashUnsupported,
     HashMismatch,
     HashUnpinned,
@@ -32,6 +32,7 @@
 from pip._internal.metadata import (
     BaseDistribution,
     get_metadata_distribution,
+    serialize_metadata,
 )
 from pip._internal.models.direct_url import ArchiveInfo
 from pip._internal.models.link import Link
@@ -230,7 +231,7 @@ class CacheableDist:
     def from_dist(cls, link: Link, dist: BaseDistribution) -> "CacheableDist":
         """Extract the serializable data necessary to generate a metadata-only dist."""
         return cls(
-            metadata=str(dist.metadata),
+            metadata=serialize_metadata(dist.metadata),
             filename=Path(link.filename),
             canonical_name=dist.canonical_name,
         )
@@ -251,7 +252,7 @@ def to_json(self) -> Dict[str, Any]:
         }
 
     @classmethod
-    def from_json(cls, args: Dict[str, Any]) -> "CacheableDist":
+    def from_json(cls, args: Dict[str, str]) -> "CacheableDist":
         return cls(
             metadata=args["metadata"],
             filename=Path(args["filename"]),
@@ -458,17 +459,10 @@ def _fetch_cached_metadata(
                     "found cached metadata for link %s at %s", req.link, f.name
                 )
                 args = json.load(f)
-                cached_dist = CacheableDist.from_json(args)
-                return cached_dist.to_dist()
-        except (OSError, json.JSONDecodeError, KeyError) as e:
-            logger.exception(
-                "error reading cached metadata for link %s at %s %s(%s)",
-                req.link,
-                cached_path,
-                e.__class__.__name__,
-                str(e),
-            )
-            raise
+            cached_dist = CacheableDist.from_json(args)
+            return cached_dist.to_dist()
+        except Exception:
+            raise CacheMetadataError(req, "error reading cached metadata")
 
     def _cache_metadata(
         self,
@@ -490,23 +484,13 @@ def _cache_metadata(
         # containing directory for the cache file exists before writing.
         os.makedirs(str(cached_path.parent), exist_ok=True)
         try:
+            cacheable_dist = CacheableDist.from_dist(req.link, metadata_dist)
+            args = cacheable_dist.to_json()
+            logger.debug("caching metadata for link %s at %s", req.link, cached_path)
             with gzip.open(cached_path, mode="wt", encoding="utf-8") as f:
-                cacheable_dist = CacheableDist.from_dist(req.link, metadata_dist)
-                args = cacheable_dist.to_json()
-                logger.debug("caching metadata for link %s at %s", req.link, f.name)
                 json.dump(args, f)
-        except (OSError, email.errors.HeaderParseError) as e:
-            # TODO: Some dists raise email.errors.HeaderParseError when calling str() or
-            # bytes() on the metadata, which is an email.Message. This is probably a bug
-            # in email parsing.
-            logger.exception(
-                "error caching metadata for dist %s from %s: %s(%s)",
-                metadata_dist,
-                req.link,
-                e.__class__.__name__,
-                str(e),
-            )
-            raise
+        except Exception:
+            raise CacheMetadataError(req, "failed to serialize metadata")
 
     def _fetch_metadata_using_link_data_attr(
         self,