Merge pull request #3222 from Syzygy2048/feature/texttiling-vocabulary-introduction

Implement vocabulary introduction for texttiling

Author: stevenbird

AUTHORS.md

@@ -311,6 +311,7 @@
 - Jose Cols <https://github.com/josecols>
 - Christopher Smith <https://github.com/smithct2>
 - Ryan Mannion <https://github.com/ryanamannion>
+- Peter Pollak <https://github.com/Syzygy2048/>
 
 ## Others whose work we've taken and included in NLTK, but who didn't directly contribute it:
 

nltk/test/unit/test_texttiling.py

@@ -0,0 +1,197 @@
+"""
+Unit tests for nltk.tokenize.texttiling.
+
+Marti A. Hearst (1997) "TextTiling: Segmenting Text into Multi-Paragraph
+Subtopic Passages", Computational Linguistics, 23(1), pp. 33-64.
+https://aclanthology.org/J97-1003.pdf
+"""
+
+from nltk.tokenize.texttiling import (
+    BLOCK_COMPARISON,
+    VOCABULARY_INTRODUCTION,
+    TextTilingTokenizer,
+)
+
+# A text with clear topic shifts for testing.
+# Topic 1: animals in nature
+# Topic 2: stock market / finance
+# Topic 3: marine biology
+# Topic 4: weather / agriculture
+MULTI_TOPIC_TEXT = (
+    """
+The quick brown fox jumped over the lazy dog. The dog was sleeping
+in the sun. It was a beautiful day outside in the forest. The fox
+continued on its way through the dense woodland. Birds were singing
+in the trees and rabbits scurried through the underbrush. The animals
+were enjoying the warm afternoon sunshine.
+
+Meanwhile, in the city, the stock market was experiencing significant
+changes. Traders were anxious about the new economic policies. The
+financial sector was particularly affected by the recent changes in
+interest rates. Banks reported lower quarterly earnings. Investors
+were pulling their money out of risky assets. The central bank was
+considering raising rates again.
+
+In other news, scientists discovered a new species of deep-sea fish.
+The fish was found at a depth of three thousand meters below the
+surface. Marine biologists were excited about the discovery. The
+specimen was brought to the laboratory for further study. Researchers
+believe the species has been living in the deep ocean for millions
+of years.
+
+The weather forecast predicted rain for the weekend across the region.
+Farmers were hoping for the precipitation to help their growing crops.
+The drought had lasted several months with no relief in sight. Water
+reservoirs were at critically low levels. Irrigation systems were
+being strained beyond their capacity.
+"""
+    * 3
+)  # repeat to get enough text for the algorithm
+
+
+class TestTextTilingBlockComparison:
+    """Tests for the block comparison method (Hearst 1997, Section 3.2)."""
+
+    def test_returns_segments(self):
+        """Block comparison should return a list of text segments."""
+        tt = TextTilingTokenizer(w=10, k=5, similarity_method=BLOCK_COMPARISON)
+        segments = tt.tokenize(MULTI_TOPIC_TEXT)
+        assert isinstance(segments, list)
+        assert len(segments) >= 1
+
+    def test_segments_cover_full_text(self):
+        """Concatenated segments should reconstruct the original text."""
+        tt = TextTilingTokenizer(w=10, k=5, similarity_method=BLOCK_COMPARISON)
+        segments = tt.tokenize(MULTI_TOPIC_TEXT)
+        assert "".join(segments) == MULTI_TOPIC_TEXT
+
+    def test_demo_mode_returns_scores(self):
+        """In demo mode, should return (gap_scores, smooth, depth, boundaries)."""
+        tt = TextTilingTokenizer(
+            w=10, k=5, similarity_method=BLOCK_COMPARISON, demo_mode=True
+        )
+        result = tt.tokenize(MULTI_TOPIC_TEXT)
+        assert len(result) == 4
+        gap_scores, smooth_scores, depth_scores, boundaries = result
+        assert len(gap_scores) > 0
+        assert len(smooth_scores) > 0
+        assert len(depth_scores) > 0
+        assert len(boundaries) > 0
+
+    def test_gap_scores_between_zero_and_one(self):
+        """Block comparison gap scores should be in [0, 1] (cosine similarity)."""
+        tt = TextTilingTokenizer(
+            w=10, k=5, similarity_method=BLOCK_COMPARISON, demo_mode=True
+        )
+        gap_scores, _, _, _ = tt.tokenize(MULTI_TOPIC_TEXT)
+        for score in gap_scores:
+            assert 0.0 <= score <= 1.0, f"Score {score} out of [0, 1] range"
+
+    def test_homogeneous_text_few_boundaries(self):
+        """Repeating the same paragraph should produce few boundaries."""
+        paragraph = "The cat sat on the mat. The dog chased the ball. "
+        # Need paragraph breaks (double newlines) for the algorithm
+        repeated = ("\n\n".join([paragraph] * 10)) + "\n\n"
+        tt = TextTilingTokenizer(w=10, k=5, similarity_method=BLOCK_COMPARISON)
+        segments = tt.tokenize(repeated)
+        # Homogeneous text should have few segments
+        assert len(segments) <= 5
+
+
+class TestTextTilingVocabIntroduction:
+    """Tests for the vocabulary introduction method (Hearst 1997, Section 3.2)."""
+
+    def test_returns_segments(self):
+        """Vocabulary introduction should return a list of text segments."""
+        tt = TextTilingTokenizer(w=10, k=5, similarity_method=VOCABULARY_INTRODUCTION)
+        segments = tt.tokenize(MULTI_TOPIC_TEXT)
+        assert isinstance(segments, list)
+        assert len(segments) >= 1
+
+    def test_segments_cover_full_text(self):
+        """Concatenated segments should reconstruct the original text."""
+        tt = TextTilingTokenizer(w=10, k=5, similarity_method=VOCABULARY_INTRODUCTION)
+        segments = tt.tokenize(MULTI_TOPIC_TEXT)
+        assert "".join(segments) == MULTI_TOPIC_TEXT
+
+    def test_demo_mode_returns_scores(self):
+        """In demo mode, should return (gap_scores, smooth, depth, boundaries)."""
+        tt = TextTilingTokenizer(
+            w=10, k=5, similarity_method=VOCABULARY_INTRODUCTION, demo_mode=True
+        )
+        result = tt.tokenize(MULTI_TOPIC_TEXT)
+        assert len(result) == 4
+        gap_scores, smooth_scores, depth_scores, boundaries = result
+        assert len(gap_scores) > 0
+        assert len(smooth_scores) > 0
+        assert len(depth_scores) > 0
+        assert len(boundaries) > 0
+
+    def test_gap_scores_non_negative(self):
+        """Vocabulary introduction scores should be non-negative."""
+        tt = TextTilingTokenizer(
+            w=10, k=5, similarity_method=VOCABULARY_INTRODUCTION, demo_mode=True
+        )
+        gap_scores, _, _, _ = tt.tokenize(MULTI_TOPIC_TEXT)
+        for score in gap_scores:
+            assert score >= 0.0, f"Score {score} is negative"
+
+    def test_homogeneous_text_low_scores(self):
+        """Repeating the same paragraph should produce low vocab intro scores."""
+        paragraph = "The cat sat on the mat. The dog chased the ball. "
+        repeated = ("\n\n".join([paragraph] * 10)) + "\n\n"
+        tt = TextTilingTokenizer(
+            w=10, k=5, similarity_method=VOCABULARY_INTRODUCTION, demo_mode=True
+        )
+        gap_scores, _, _, _ = tt.tokenize(repeated)
+        # After the first few windows, no new vocabulary is introduced
+        # so later scores should be near zero
+        later_scores = gap_scores[len(gap_scores) // 2 :]
+        avg_later = sum(later_scores) / len(later_scores) if later_scores else 0
+        assert (
+            avg_later < 0.15
+        ), f"Average later score {avg_later} too high for repeated text"
+
+
+class TestTextTilingCommon:
+    """Tests common to both methods."""
+
+    def test_both_methods_find_boundaries(self):
+        """Both methods should find at least one boundary in multi-topic text."""
+        for method in [BLOCK_COMPARISON, VOCABULARY_INTRODUCTION]:
+            tt = TextTilingTokenizer(w=10, k=5, similarity_method=method)
+            segments = tt.tokenize(MULTI_TOPIC_TEXT)
+            assert (
+                len(segments) > 1
+            ), f"Method {method} found no boundaries in multi-topic text"
+
+    def test_invalid_similarity_method(self):
+        """Invalid similarity method should raise ValueError."""
+        tt = TextTilingTokenizer(similarity_method="invalid")
+        try:
+            tt.tokenize(MULTI_TOPIC_TEXT)
+            assert False, "Expected ValueError"
+        except ValueError:
+            pass
+
+    def test_invalid_smoothing_method(self):
+        """Invalid smoothing method should raise ValueError."""
+        tt = TextTilingTokenizer(smoothing_method="invalid")
+        try:
+            tt.tokenize(MULTI_TOPIC_TEXT)
+            assert False, "Expected ValueError"
+        except ValueError:
+            pass
+
+    def test_boundary_count_matches_segments(self):
+        """Number of boundaries should be number of segments minus 1."""
+        for method in [BLOCK_COMPARISON, VOCABULARY_INTRODUCTION]:
+            tt = TextTilingTokenizer(w=10, k=5, similarity_method=method)
+            segments = tt.tokenize(MULTI_TOPIC_TEXT)
+            tt_demo = TextTilingTokenizer(
+                w=10, k=5, similarity_method=method, demo_mode=True
+            )
+            _, _, _, boundaries = tt_demo.tokenize(MULTI_TOPIC_TEXT)
+            num_boundaries = sum(boundaries)
+            # segments = boundaries + 1 (approximately, due to normalization)
+            assert num_boundaries >= len(segments) - 2

nltk/tokenize/texttiling.py

@@ -16,7 +16,8 @@
 
 from nltk.tokenize.api import TokenizerI
 
-BLOCK_COMPARISON, VOCABULARY_INTRODUCTION = 0, 1
+BLOCK_COMPARISON = "block_comparison"
+VOCABULARY_INTRODUCTION = "vocabulary_introduction"
 LC, HC = 0, 1
 DEFAULT_SMOOTHING = [0]
 
@@ -118,7 +119,7 @@ def tokenize(self, text):
         if self.similarity_method == BLOCK_COMPARISON:
             gap_scores = self._block_comparison(tokseqs, token_table)
         elif self.similarity_method == VOCABULARY_INTRODUCTION:
-            raise NotImplementedError("Vocabulary introduction not implemented")
+            gap_scores = self._vocabulary_introduction(tokseqs)
         else:
             raise ValueError(
                 f"Similarity method {self.similarity_method} not recognized"
@@ -157,6 +158,83 @@ def tokenize(self, text):
             return gap_scores, smooth_scores, depth_scores, segment_boundaries
         return segmented_text
 
+    def _vocabulary_introduction(self, tokseqs):
+        """Compute gap scores using the Vocabulary Introduction method.
+
+        From Marti A. Hearst (1997) "TextTiling: Segmenting Text into
+        Multi-Paragraph Subtopic Passages", Computational Linguistics,
+        23(1), pp. 33-64. https://aclanthology.org/J97-1003.pdf
+
+        Section 3.2:
+
+        The idea behind this approach is that the introduction of a new
+        topic in a text is signaled by the distribution of new vocabulary
+        items. For each pseudosentence gap, we count the number of new
+        word types appearing on each side of the gap that have not been
+        seen in earlier pseudosentences on that side.
+
+        Schematically (adapted from Fig 3, Hearst 1997)::
+
+            pseudosentences:  [ s1 ] [ s2 ] [ s3 ] [ s4 ] [ s5 ] ...
+                                          ^
+                                       gap at i=2
+                                          |
+              left side of gap:  s1, s2   |   right side: s3, s4, s5, ...
+              (scan left->right)          |   (scan right->left)
+                                          |
+              new_L(i) = words in s_i     |   new_R(i) = words in s_{i+1}
+                not seen in s1..s_{i-1}   |   not seen in s_{i+2}..s_N
+
+        The score for each gap is::
+
+            score(i) = ( new_L(i) + new_R(i) ) / (2 * w)
+
+        where ``w`` is the pseudosentence size, so ``2 * w`` normalizes
+        by the maximum possible number of new word types across both
+        sides of the gap.
+
+        :param tokseqs: list of TokenSequence objects
+        :return: list of gap scores (length = len(tokseqs) - 1)
+        """
+        n = len(tokseqs)
+        if n < 2:
+            return []
+
+        # Extract the word type sets for each pseudosentence
+        tokseq_sets = [{token for token, _ in seq.wrdindex_list} for seq in tokseqs]
+
+        # Normalization factor (Section 3.2)
+        norm = self.w * 2
+
+        # Scan left-to-right: for each position i, count how many words
+        # in tokseq_sets[i] are new (not seen in any s_0..s_{i-1}).
+        new_left = []
+        seen_left = set()
+        for i in range(n):
+            new_count = len(tokseq_sets[i] - seen_left)
+            new_left.append(new_count)
+            seen_left |= tokseq_sets[i]
+
+        # Scan right-to-left: for each position i, count how many words
+        # in tokseq_sets[i] are new (not seen in any s_{i+1}..s_{N-1}).
+        new_right = [0] * n
+        seen_right = set()
+        for i in range(n - 1, -1, -1):
+            new_count = len(tokseq_sets[i] - seen_right)
+            new_right[i] = new_count
+            seen_right |= tokseq_sets[i]
+
+        # Score each gap between pseudosentence i and i+1.
+        # The left contribution comes from the pseudosentence just
+        # before the gap (new_left[i]), and the right contribution
+        # from the pseudosentence just after (new_right[i+1]).
+        gap_scores = []
+        for i in range(n - 1):
+            score = (new_left[i] + new_right[i + 1]) / norm
+            gap_scores.append(score)
+
+        return gap_scores
+
     def _block_comparison(self, tokseqs, token_table):
         """Implements the block comparison method"""
 
@@ -429,10 +507,12 @@ def smooth(x, window_len=11, window="flat"):
     """
 
     if x.ndim != 1:
-        raise ValueError("smooth only accepts 1 dimension arrays.")
+        raise ValueError(f"smooth only accepts 1 dimension arrays, was {x.ndim}.")
 
     if x.size < window_len:
-        raise ValueError("Input vector needs to be bigger than window size.")
+        raise ValueError(
+            f"Input vector ({len(x)}) needs to be bigger than window size ({window_len})."
+        )
 
     if window_len < 3:
         return x
@@ -455,15 +535,17 @@ def smooth(x, window_len=11, window="flat"):
     return y[window_len - 1 : -window_len + 1]
 
 
-def demo(text=None):
+def demo(text=None, similarity_method=BLOCK_COMPARISON):
     from matplotlib import pylab
 
     from nltk.corpus import brown
 
-    tt = TextTilingTokenizer(demo_mode=True)
+    pylab.figure()
+    tt = TextTilingTokenizer(demo_mode=True, similarity_method=similarity_method)
     if text is None:
         text = brown.raw()[:10000]
     s, ss, d, b = tt.tokenize(text)
+    pylab.title(f"TextTiling: {similarity_method}")
     pylab.xlabel("Sentence Gap index")
     pylab.ylabel("Gap Scores")
     pylab.plot(range(len(s)), s, label="Gap Scores")