Indexing

Support re2match using btree to find prefix

Support GIN indexing with @~ operator

Author: serprex

.github/workflows/ci.yml

@@ -26,6 +26,6 @@ jobs:
     runs-on: ubuntu-latest
     container: pgxn/pgxn-tools
     steps:
-      - run: pg-start ${{ matrix.pg }} libre2-dev
+      - run: pg-start ${{ matrix.pg }} libre2-dev libicu-dev
       - uses: actions/checkout@v7
       - run: pg-build-test

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v7
       - name: Start Postgres
-        run: pg-start 19 libre2-dev
+        run: pg-start 19 libre2-dev libicu-dev
       - name: Bundle the Release
         id: bundle
         run: pgxn-bundle

CHANGELOG.md

@@ -14,6 +14,22 @@ All notable changes to this project will be documented in this file. It uses the
 *   Added `re2_version()`, which returns the full semantic version. This is
     the same value visible in `pg_get_loaded_modules()`, but available in
     Postgres versions prior to 18, and without having to load re2 first.
+*   Index support.
+    *   `WHERE re2match(col, '^order_2025')` a pattern starting with `^` and a
+        fixed prefix now reads only matching range of an existing b-tree index.
+        See [Index Support](doc/re2.md#index-support).
+    *   Added `@~` match operator (same as `re2match`) and `gin_re2_ops`
+        operator class. `CREATE INDEX ... USING gin (col gin_re2_ops)` lets
+        `WHERE col @~ pattern` only visit rows containing required trigrams.
+    *   Row-count estimates for `re2match` and `@~` filters with a constant
+        pattern now come from testing the pattern against column statistics
+        instead of a fixed guess, so the planner picks better plans around
+        such filters.
+
+### 📔 Notes
+
+*   Run `ALTER EXTENSION re2 UPDATE TO '0.4'` to expose the new operator and
+    operator class on existing databases.
 
   [v0.4.0]: https://github.com/clickhouse/pg_re2/compare/v0.3.0...v0.4.0
 

Makefile

@@ -7,7 +7,7 @@ DISTVERSION  = $(shell grep -m 1 '^[[:space:]]\{2\}"version":' META.json | \
 
 DATA         = $(wildcard sql/$(EXTENSION)--*.sql)
 MODULE_big   = $(EXTENSION)
-OBJS         = src/pg_re2.o src/re2_cache.o src/re2_wrapper.o
+OBJS         = src/pg_re2.o src/pg_re2_index.o src/re2_cache.o src/re2_wrapper.o
 
 PG_CONFIG   ?= pg_config
 PG_CXXFLAGS  = -std=c++17
@@ -28,17 +28,17 @@ $(OBJS): src/version.h
 src/version.h: src/version.h.in
 	sed -e 's,__VERSION__,$(DISTVERSION),g' $< > $@
 
-.PHONY: format
-format: src/*.c src/*.h src/*.cpp
-	clang-format -i $^
+CLANG_FORMAT ?= clang-format
 
-.PHONY: lint
-lint: src/*.c src/*.h src/*.cpp
-	clang-format --dry-run --Werror $^
+.PHONY: format format22
+format22: CLANG_FORMAT = clang-format-22
+format format22: src/*.c src/*.h src/*.cpp
+	$(CLANG_FORMAT) -i $^
 
-.PHONY: lint22
-lint22: src/*.c src/*.h src/*.cpp
-	clang-format-22 --dry-run --Werror $^
+.PHONY: lint lint22
+lint22: CLANG_FORMAT = clang-format-22
+lint lint22: src/*.c src/*.h src/*.cpp
+	$(CLANG_FORMAT) --dry-run --Werror $^
 
 .PHONY: bench
 bench:

README.md

@@ -74,6 +74,15 @@ make CPPFLAGS=-I/opt/homebrew/include \
 
 If you encounter an error such as:
 
+```
+utils/pg_locale.h:24:10: fatal error: 'unicode/ucol.h' file not found
+```
+
+Install the ICU development headers Postgres was built against, e.g.
+`libicu-dev` on Debian and Ubuntu.
+
+If you encounter an error such as:
+
 ```
 ERROR:  must be owner of database regression
 ```

benchmark/README.md

@@ -22,24 +22,61 @@ Data is 10000 rows of:
 - `logline` ~200 chars
 - `longtext` ~2000 chars (400 words)
 
+Index scans
+-----------
+
+`re2` also speeds up `re2match` through two index mechanisms (see
+[Index Support]). This compares each against the equivalent PostgreSQL
+index scan over a separate 100000-row table.
+
+| Mechanism           | re2                          | postgres                     |
+| ------------------- | ---------------------------- | ---------------------------- |
+| b-tree prefix range | `re2match(col, '^lit')`      | `col ~ '^lit'`               |
+| GIN trigram         | `col @~ pat` (`gin_re2_ops`) | `col ~ pat` (`gin_trgm_ops`) |
+
+![Index benchmark](graph_index.png)
+
+| Category | Pattern                       | rows  | re2     | postgres | re2 vs postgres |
+| -------- | ----------------------------- | ----- | ------- | -------- | --------------- |
+| btree    | `^user5`                      | 11111 | 1.8 ms  | 3.5 ms   | 1.9x faster     |
+| btree    | `^user12[0-9]`                | 1110  | 0.21 ms | 0.43 ms  | 2.0x faster     |
+| gin      | `error_code=123`              | 100   | 3.3 ms  | 3.6 ms   | 1.1x faster     |
+| gin      | `error_code=(100\|200\|300)`  | 301   | 3.5 ms  | 4.9 ms   | 1.4x faster     |
+
+The two GIN opclasses extract keys differently. `pg_trgm` builds trigrams from
+alphanumeric words only (never spanning `_`, `=`, …) and prunes extracted
+trigrams under a fixed penalty budget tuned for natural-language text;
+`gin_re2_ops` keeps every byte trigram of each literal atom RE2's `FilteredRE2`
+requires. On punctuated machine-text patterns (e.g. `error_code=42[0-9]` over
+loglines where `error_code=` appears in every row) pruning can leave `pg_trgm`
+with only ubiquitous trigrams, degenerating to a full-index scan while
+`gin_re2_ops` stays selective, an order of magnitude faster. On plain-word
+patterns both extract similar keys and `pg_trgm`'s cheaper consistent check
+can win (see `error_code=123` above).
+
 Methodology
 -----------
 
 - JIT and query parallelism disabled to compare single-thread engine throughput reliably
 - `gen_graph.py` takes the median time per (pattern, engine) across all iterations
+- Index scans use a `text_pattern_ops` b-tree and two GIN indexes on one table;
+  `enable_seqscan` is off there so both engines are measured on their index
 
 Running
 -------
 
-Requires `re2` extension installed (see [README]) and PostgreSQL 15+ for builtin comparisons
+Requires `re2` (see [README]) and PostgreSQL 15+ for builtin comparisons.
+The index-scan section additionally needs the `pg_trgm` contrib extension;
+`setup.sql` creates it.
 
 Connection uses libpq environment variables; override the `psql` binary with
 `PSQL`:
 
 ``` sh
 PGDATABASE=mydb ./run_bench.sh        # 5 iterations (default)
 PGDATABASE=mydb ./run_bench.sh 10     # 10 iterations
-./gen_graph.py                        # regenerate graph.png from results.csv
+./gen_graph.py                        # regenerate graph.png & graph_index.png
 ```
 
   [README]: ../README.md
+  [Index Support]: ../doc/re2.md#index-support

benchmark/bench.sql

@@ -262,3 +262,60 @@ SELECT 'count_matches', 'words_longtext', 'pg_builtin',
   (SELECT sum(regexp_count(longtext, '\w+')) FROM bench_data WHERE id <= 2500),
   (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
 FROM (SELECT clock_timestamp() AS ts) t;
+
+-- ============================================================
+-- 7. INDEX SCAN: re2 index support vs postgres index scan
+--   idx_btree  re2match(col,'^lit') planner support -> text_pattern_ops range,
+--              vs postgres' own '~' prefix support on the same index
+--   idx_gin    col @~ pat (gin_re2_ops, RE2 FilteredRE2 atoms)
+--              vs col ~ pat (pg_trgm gin_trgm_ops)
+-- enable_seqscan off so both engines are measured on their index, not a scan
+-- ============================================================
+SET enable_seqscan = off;
+
+-- 7a. b-tree literal prefix (~11k rows in range, recheck cost visible)
+SELECT 'idx_btree', 'prefix_literal', 're2',
+  (SELECT count(*) FROM bench_index_data WHERE re2match(email, '^user5')),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+SELECT 'idx_btree', 'prefix_literal', 'pg_builtin',
+  (SELECT count(*) FROM bench_index_data WHERE email ~ '^user5'),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+-- 7b. b-tree literal prefix + bounded char-class tail (prefix still extractable)
+SELECT 'idx_btree', 'prefix_charclass', 're2',
+  (SELECT count(*) FROM bench_index_data WHERE re2match(email, '^user12[0-9]')),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+SELECT 'idx_btree', 'prefix_charclass', 'pg_builtin',
+  (SELECT count(*) FROM bench_index_data WHERE email ~ '^user12[0-9]'),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+-- 7c. GIN required literal atom (re2 keeps a tight candidate set; pg_trgm wins
+-- here because its per-candidate consistent check is far cheaper)
+SELECT 'idx_gin', 'literal', 're2',
+  (SELECT count(*) FROM bench_index_data WHERE logline @~ 'error_code=123'),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+SELECT 'idx_gin', 'literal', 'pg_builtin',
+  (SELECT count(*) FROM bench_index_data WHERE logline ~ 'error_code=123'),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+-- 7d. GIN alternation (atoms on each branch)
+SELECT 'idx_gin', 'alternation', 're2',
+  (SELECT count(*) FROM bench_index_data WHERE logline @~ 'error_code=(100|200|300)'),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+SELECT 'idx_gin', 'alternation', 'pg_builtin',
+  (SELECT count(*) FROM bench_index_data WHERE logline ~ 'error_code=(100|200|300)'),
+  (extract(epoch FROM clock_timestamp() - ts) * 1000)::numeric(12,2)
+FROM (SELECT clock_timestamp() AS ts) t;
+
+RESET enable_seqscan;

benchmark/gen_graph.py

@@ -184,8 +184,10 @@ def chunk(kind, data):
         f.write(png)
 
 
-def plot_speedup(tests, filename='graph.png'):
-    """Render horizontal speedup bars"""
+def plot_speedup(tests, filename='graph.png',
+                 title='re2 speedup over postgresql builtin regex',
+                 axis_cap=None):
+    """Render horizontal speedup bars; axis_cap clips outliers to given max"""
     rows = _rows_by_speedup(tests)
     path = os.path.join(OUT_DIR, filename)
 
@@ -201,7 +203,10 @@ def plot_speedup(tests, filename='graph.png'):
     display_rows = list(reversed(rows))
     speedups = [r[3] for r in rows]
     max_speedup = max(speedups)
-    ticks = _tick_values(max_speedup * 1.05)
+    target = max_speedup * 1.05
+    if axis_cap:
+        target = min(target, axis_cap)
+    ticks = _tick_values(target)
     axis_max = max(ticks)
 
     label_width = max(_text_width(label, 2) for label, _, _, _ in rows)
@@ -224,7 +229,7 @@ def plot_speedup(tests, filename='graph.png'):
         height,
         24,
         24,
-        're2 speedup over postgresql builtin regex',
+        title,
         FG,
         3,
     )
@@ -251,8 +256,10 @@ def plot_speedup(tests, filename='graph.png'):
 
         value = f'{speedup:.1f}x'
         value_width = _text_width(value, 2)
-        value_color = BG if value_width + 16 <= bar_width else FG
-        _draw_text(pixels, width, height, left + 8, y + 2, value, value_color, 2)
+        if value_width + 16 <= bar_width:
+            _draw_text(pixels, width, height, left + 8, y + 2, value, BG, 2)
+        else:
+            _draw_text(pixels, width, height, left + bar_width + 6, y + 2, value, FG, 2)
 
     if 1 <= axis_max:
         x = left + plot_width / axis_max
@@ -276,8 +283,21 @@ def plot_speedup(tests, filename='graph.png'):
     print(f"Saved {path}")
 
 
+def _split_index(tests):
+    """Partition tests into (throughput, index) by category prefix"""
+    throughput = {}
+    index = {}
+    for key, engines in tests.items():
+        (index if key[0].startswith('idx_') else throughput)[key] = engines
+    return throughput, index
+
+
 if __name__ == '__main__':
     data = load_results()
     tests = build_comparison_data(data)
-    plot_speedup(tests)
+    throughput, index = _split_index(tests)
+    plot_speedup(throughput, 'graph.png')
+    if index:
+        plot_speedup(index, 'graph_index.png',
+                     're2 index scan speedup over postgresql', axis_cap=20)
     print("Done.")

benchmark/graph_index.png

@@ -18,7 +18,7 @@ for i in $(seq 1 "$ITERATIONS"); do
     echo "=== Iteration $i/$ITERATIONS ===" >&2
     "$PSQL" -X -q -f "$SCRIPT_DIR/bench.sql" | while IFS= read -r line; do
         case "$line" in
-            match,*|extract,*|extract_all,*|extract_all_unnest,*|replace_one,*|replace_all,*|count_matches,*)
+            match,*|extract,*|extract_all,*|extract_all_unnest,*|replace_one,*|replace_all,*|count_matches,*|idx_btree,*|idx_gin,*)
                 echo "${line},${i}" ;;
         esac
     done >> "$OUTFILE"