fix(message-extractor): improve handling of forwarded messages and enhance text cleanup

Author: wmontwe

legacy/core/src/main/java/com/fsck/k9/message/extractors/PreviewTextExtractor.kt

@@ -7,6 +7,7 @@ import com.fsck.k9.message.html.EmailSection
 import com.fsck.k9.message.html.EmailSectionExtractor
 import com.fsck.k9.message.html.HtmlConverter
 
+@Suppress("TooManyFunctions")
 internal class PreviewTextExtractor {
     @Throws(PreviewExtractionException::class)
     fun extractPreview(textPart: Part): String {
@@ -32,18 +33,13 @@ internal class PreviewTextExtractor {
         intermediateText = stripSignature(intermediateText)
         intermediateText = extractUnquotedText(intermediateText)
 
-        // try to remove lines of dashes in the preview
-        intermediateText = intermediateText.replace("(?m)^----.*?$".toRegex(), "")
-        // Remove horizontal rules.
-        intermediateText = intermediateText.replace("\\s*([-=_]{30,}+)\\s*".toRegex(), " ")
-
+        // Run line-based cleanup before HTML normalization, so that we don't remove line breaks in HTML
+        intermediateText = stripLineBasedArtifacts(intermediateText)
         // Always parse the text as HTML, independently of the mimetype
         intermediateText = HtmlConverter.htmlToText(intermediateText)
-        // Remove parsed HTML links/images "<url>", "( url )", "(url)", etc.
-        intermediateText = intermediateText.replace("[(<]\\s?https?://\\S+[^)>]\\s?[>)]".toRegex(), " ")
-
-        // Remove invisible formatting characters that can otherwise dominate previews.
-        intermediateText = intermediateText.replace("[\\u034F\\u200B-\\u200D\\uFEFF]".toRegex(), "")
+        intermediateText = stripLineBasedArtifacts(intermediateText)
+        intermediateText = removeParsedHtmlUrls(intermediateText)
+        intermediateText = removeInvisibleFormattingCharacters(intermediateText)
 
         // URLs in the preview should just be shown as "..." - They're not
         // clickable and they usually overwhelm the preview
@@ -62,6 +58,45 @@ internal class PreviewTextExtractor {
         }
     }
 
+    private fun stripLineBasedArtifacts(text: String): String {
+        var strippedText = text
+
+        strippedText = stripForwardedMessageMetadata(strippedText)
+        strippedText = stripDashLines(strippedText)
+        strippedText = stripHorizontalRules(strippedText)
+
+        return strippedText
+    }
+
+    private fun stripForwardedMessageMetadata(text: String): String {
+        return REGEX_FORWARDED_MESSAGE_HEADER_BLOCK.replace(text) { matchResult ->
+            val hasContentBefore = text.substring(0, matchResult.range.first).isNotBlank()
+            val hasContentAfter = text.substring(matchResult.range.last + 1).isNotBlank()
+
+            if (hasContentBefore && hasContentAfter) {
+                " $PREVIEW_SECTION_SEPARATOR "
+            } else {
+                " "
+            }
+        }
+    }
+
+    private fun stripDashLines(text: String): String {
+        return text.replace(REGEX_DASH_LINE, "")
+    }
+
+    private fun stripHorizontalRules(text: String): String {
+        return text.replace(REGEX_HORIZONTAL_RULE, " ")
+    }
+
+    private fun removeParsedHtmlUrls(text: String): String {
+        return text.replace(REGEX_PARSED_HTML_URL, " ")
+    }
+
+    private fun removeInvisibleFormattingCharacters(text: String): String {
+        return text.replace(REGEX_INVISIBLE_FORMATTING_CHARACTERS, "")
+    }
+
     private fun normalizeLineBreaks(text: String) = text.replace(REGEX_CRLF, "\n")
 
     private fun stripSignature(text: String): String {
@@ -129,7 +164,17 @@ internal class PreviewTextExtractor {
     companion object {
         private const val MAX_PREVIEW_LENGTH = 512
         private const val MAX_CHARACTERS_CHECKED_FOR_PREVIEW = 8192L
+        private const val PREVIEW_SECTION_SEPARATOR = "[…]"
 
         private val REGEX_CRLF = "(\\r\\n|\\r)".toRegex()
+        private val REGEX_DASH_LINE = "(?m)^-{4,}\\s*$".toRegex()
+        private val REGEX_HORIZONTAL_RULE = "\\s*([-=_]{30,}+)\\s*".toRegex()
+        private val REGEX_PARSED_HTML_URL = "[(<]\\s?https?://\\S+[^)>]\\s?[>)]".toRegex()
+        private val REGEX_INVISIBLE_FORMATTING_CHARACTERS = "[\\u034F\\u200B-\\u200D\\uFEFF]".toRegex()
+        private val REGEX_FORWARDED_MESSAGE_HEADER_BLOCK = (
+            "(?im)^[\\t -]*Original Message[\\t -]*\\n" +
+                "(?:[\\t ]*[A-Za-z][A-Za-z-]*:.*\\n)+" +
+                "[\\t ]*\\n?"
+            ).toRegex()
     }
 }

legacy/core/src/test/java/com/fsck/k9/message/extractors/PreviewTextExtractorTest.kt

@@ -222,6 +222,52 @@ class PreviewTextExtractorTest {
         assertThat(preview).isEqualTo("some image:")
     }
 
+    @Test
+    fun extractPreview_forwardedMessage() {
+        val text =
+            """
+            |Here is the forwarded message:
+            |
+            |-----Original Message-----
+            |From: alice@example.com
+            |Sent: Monday, January 1, 2024 10:00 AM
+            |To: bob@example.com
+            |Subject: Hello
+            |
+            |This is the original content.
+            """.trimMargin()
+        val part = MessageCreationHelper.createTextPart("text/plain", text)
+
+        val preview = previewTextExtractor.extractPreview(part)
+
+        assertThat(preview).isEqualTo("Here is the forwarded message: […] This is the original content.")
+    }
+
+    @Test
+    fun extractPreview_withHtmlForwardedMessageAsTextPlain() {
+        val text =
+            """
+            |<html>
+            |<body>
+            |Here is the forwarded message:<br>
+            |<br>
+            |-----Original Message-----<br>
+            |From: alice@example.com<br>
+            |Sent: Monday, January 1, 2024 10:00 AM<br>
+            |To: bob@example.com<br>
+            |Subject: Hello<br>
+            |<br>
+            |This is the original content.
+            |</body>
+            |</html>
+            """.trimMargin()
+        val part = MessageCreationHelper.createTextPart("text/plain", text)
+
+        val preview = previewTextExtractor.extractPreview(part)
+
+        assertThat(preview).isEqualTo("Here is the forwarded message: […] This is the original content.")
+    }
+
     @Test
     fun extractPreview_shouldCollapseAndTrimWhitespace() {
         val text = " whitespace     is\t\tfun  "