feat(chunker): wasm/wazero tree-sitter backend — mmap memory, minified skip, doc-comment attachment

Replace gotreesitter with the official tree-sitter C runtime + 31 grammars
compiled to one wasm32-wasi module (ts-core.wasm.br, brotli ~3MB) driven via
wazero. No cgo: traps are contained (parse falls back to sliding window, the
process survives), and the binary stays CGO_ENABLED=0.

Memory design (measured on the prod-shaped churn workload):
- linear memory is mmap-backed (experimental.WithMemoryAllocator) instead of
  wazero's default Go-heap append-grow: no realloc-copy garbage on growth and
  munmap-on-close returns recycled instances' memory to the OS immediately.
  Churn heapSys 1135→391MB, peak RSS 1070→535MB; full-repo chunking peak RSS
  1516→787MB.
- engine pool: hard concurrency cap (dashboard-tunable), 256MiB per-instance
  linear-memory ceiling (2× headroom over the worst measured instance at the
  indexer's 512KiB file cap), high-water-mark recycling, 1 idle instance.

Chunker quality fixes:
- minified/bundled js/ts/css (.min., .bundle.js, >2KiB lines) skip the parser
  straight to sliding window — the pathological input class that ballooned
  instances for near-zero semantic value.
- a declaration's doc comment now attaches to its chunk (language-agnostic via
  tree-sitter's extra flag + same-row wrapper climb; verified for Go, TS, C,
  Python, Rust, Java). Generated files stop spraying comment-only micro
  chunks: openapi.gen.go 893→517 chunks, median 114→256B, symbols/refs
  byte-identical.

Memory-stress harnesses are committed but gated behind CIX_MEMSTRESS=1.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: dvcdsys

README.md

@@ -78,7 +78,7 @@ Grep and fuzzy file search work fine for small projects. At scale they break dow
 │  └── embedded Swagger UI                                        │
 │                                                                 │
 │  Indexing pipeline                                              │
-│  ├── gotreesitter (AST chunking, 200+ languages)                │
+│  ├── tree-sitter/wasm (AST chunking, 31 langs)  (wazero)        │
 │  ├── llama-server sidecar (Unix socket → CodeRankEmbed Q8 GGUF) │
 │  ├── chromem-go (cosine similarity vector store)                │
 │  ├── SQLite FTS5 chunk mirror (BM25 — powers hybrid workspace)  │

poc/wasm-treesitter/.gitignore

@@ -0,0 +1,3 @@
+# build artifact — rebuilt by build.sh; the committed module lives as
+# server/internal/chunker/tswasm/ts-core.wasm.br (brotli)
+ts-core.wasm

poc/wasm-treesitter/build.sh

@@ -1,40 +1,108 @@
 #!/usr/bin/env bash
-# Builds ts-ts.wasm: the OFFICIAL tree-sitter C runtime + the official
-# TypeScript grammar, compiled to a standalone wasm32-wasi reactor module.
-# No emscripten, no JS glue, no third-party Go host — just official C sources
-# driven from Go via wazero (see wasmts.go).
+# Builds ts-core.wasm: the OFFICIAL tree-sitter C runtime + the base grammars +
+# our host_extra.c (the batched ts_dump_tree walk), compiled to ONE standalone
+# wasm32-wasi reactor module via `zig cc`. No emscripten, no JS glue.
 #
-# Requires: zig (provides clang + wasi-libc cross-compilation), git.
-#   brew install zig
+# Requires: zig (clang + wasi-libc cross-compile), git, and tree-sitter CLI (only
+# for grammars whose repo ships no committed parser.c — gen=1 rows).
 #
-# Key point: the only wasmtime-dependent part of the runtime (wasm_store.c) is
-# guarded by `#ifdef TREE_SITTER_FEATURE_WASM`, which we do NOT define — so the
-# stock amalgamation (lib/src/lib.c) compiles to wasi cleanly with no stubs.
+# For wasm we compile each grammar IN PLACE from a full clone, so relative
+# includes (e.g. typescript's ../../common/scanner.h) and src-root headers (html
+# tag.h, haskell unicode.h) resolve naturally — none of the vendor.sh copy/rewrite
+# dance is needed. Quirks that remain: SHA pins (dart), `tree-sitter generate`
+# (sql), and a 2nd grammar from one repo (tsx). See plan §6.1.
 set -euo pipefail
 cd "$(dirname "$0")"
 
-TS_VERSION="${TS_VERSION:-v0.25.10}"          # tree-sitter runtime
-TS_TS_VERSION="${TS_TS_VERSION:-v0.23.2}"     # tree-sitter-typescript grammar
+TS_VERSION="${TS_VERSION:-v0.25.10}"
+OUT="${OUT:-ts-core.wasm}"
 WORK="$(mktemp -d)"
 trap 'rm -rf "$WORK"' EXIT
 
-git clone --depth 1 --branch "$TS_VERSION"    https://github.com/tree-sitter/tree-sitter            "$WORK/tree-sitter"
-git clone --depth 1 --branch "$TS_TS_VERSION" https://github.com/tree-sitter/tree-sitter-typescript "$WORK/ts-typescript"
+# id  repo  ref  srcsubdir  [gen]
+GRAMMARS=(
+  "python      tree-sitter/tree-sitter-python      v0.25.0 src"
+  "typescript  tree-sitter/tree-sitter-typescript  v0.23.2 typescript/src"
+  "tsx         tree-sitter/tree-sitter-typescript  v0.23.2 tsx/src"
+  "javascript  tree-sitter/tree-sitter-javascript  v0.25.0 src"
+  "go          tree-sitter/tree-sitter-go          v0.25.0 src"
+  "rust        tree-sitter/tree-sitter-rust        v0.24.2 src"
+  "java        tree-sitter/tree-sitter-java        v0.23.5 src"
+  "c           tree-sitter/tree-sitter-c           v0.24.2 src"
+  "cpp         tree-sitter/tree-sitter-cpp         v0.23.4 src"
+  "ruby        tree-sitter/tree-sitter-ruby        v0.23.1 src"
+  "c_sharp     tree-sitter/tree-sitter-c-sharp     v0.23.5 src"
+  "php         tree-sitter/tree-sitter-php         v0.24.2 php/src"
+  "swift       alex-pinkus/tree-sitter-swift       0.7.3-with-generated-files src"
+  "kotlin      tree-sitter-grammars/tree-sitter-kotlin v1.1.0 src"
+  "scala       tree-sitter/tree-sitter-scala       v0.26.0 src"
+  "bash        tree-sitter/tree-sitter-bash        v0.25.1 src"
+  "lua         tree-sitter-grammars/tree-sitter-lua v0.5.0 src"
+  "dart        UserNobody14/tree-sitter-dart       a9bdfa3 src"
+  "r           r-lib/tree-sitter-r                 v1.2.0 src"
+  "objc        tree-sitter-grammars/tree-sitter-objc v3.0.2 src"
+  "html        tree-sitter/tree-sitter-html        v0.23.2 src"
+  "css         tree-sitter/tree-sitter-css         v0.25.0 src"
+  "scss        tree-sitter-grammars/tree-sitter-scss v1.0.0 src"
+  "sql         DerekStride/tree-sitter-sql         v0.3.11 src 1"
+  "markdown    tree-sitter-grammars/tree-sitter-markdown v0.5.3 tree-sitter-markdown/src"
+  "zig         tree-sitter-grammars/tree-sitter-zig v1.1.2 src"
+  "julia       tree-sitter/tree-sitter-julia       v0.25.0 src"
+  "fortran     stadelmanma/tree-sitter-fortran     v0.6.0 src"
+  "haskell     tree-sitter/tree-sitter-haskell     v0.23.1 src"
+  "ocaml       tree-sitter/tree-sitter-ocaml       v0.25.0 grammars/ocaml/src"
+  "solidity    JoranHonig/tree-sitter-solidity     v1.2.13 src"
+)
 
+clone() { # repo ref dest  — tag/branch fast path, SHA fallback
+  local repo="$1" ref="$2" dest="$3"
+  git clone --depth 1 --branch "$ref" "https://github.com/$repo" "$dest" >/dev/null 2>&1 && return 0
+  git clone "https://github.com/$repo" "$dest" >/dev/null 2>&1 || return 1
+  git -C "$dest" checkout "$ref" >/dev/null 2>&1
+}
+
+echo "→ tree-sitter runtime $TS_VERSION"
+git clone --depth 1 --branch "$TS_VERSION" https://github.com/tree-sitter/tree-sitter "$WORK/tree-sitter" 2>/dev/null
+
+SRCS=( "$WORK/tree-sitter/lib/src/lib.c" "csrc/host_extra.c" )
+INCS=( -I "$WORK/tree-sitter/lib/include" -I "$WORK/tree-sitter/lib/src" )
+EXPORTS=()
+BUILT=() ; FAILED=()
+
+for row in "${GRAMMARS[@]}"; do
+  read -r id repo ref sub gen <<<"$row"
+  printf '  %-12s %s@%s ' "$id" "$repo" "$ref"
+  if ! clone "$repo" "$ref" "$WORK/$id"; then echo "CLONE FAIL"; FAILED+=("$id"); continue; fi
+  gsrc="$WORK/$id/$sub"
+  if [ "${gen:-0}" = "1" ] && [ ! -f "$gsrc/parser.c" ]; then
+    ( cd "$WORK/$id" && tree-sitter generate >/dev/null 2>&1 ) || true
+  fi
+  if [ ! -f "$gsrc/parser.c" ]; then echo "NO parser.c"; FAILED+=("$id"); continue; fi
+  SRCS+=( "$gsrc/parser.c" )
+  [ -f "$gsrc/scanner.c" ]  && SRCS+=( "$gsrc/scanner.c" )
+  [ -f "$gsrc/scanner.cc" ] && SRCS+=( "$gsrc/scanner.cc" )
+  INCS+=( -I "$gsrc" )
+  EXPORTS+=( -Wl,--export=tree_sitter_$id )
+  BUILT+=("$id")
+  echo "ok"
+done
+
+echo "→ compiling ${#SRCS[@]} sources, ${#BUILT[@]} grammars → $OUT"
 zig cc --target=wasm32-wasi-musl -mexec-model=reactor \
-  -I "$WORK/tree-sitter/lib/include" -I "$WORK/tree-sitter/lib/src" \
-  -I "$WORK/ts-typescript/typescript/src" \
-  "$WORK/tree-sitter/lib/src/lib.c" \
-  "$WORK/ts-typescript/typescript/src/parser.c" \
-  "$WORK/ts-typescript/typescript/src/scanner.c" \
-  -o ts-ts.wasm -Oz -fPIC -Wl,--no-entry -Wl,--strip-debug \
+  "${INCS[@]}" "${SRCS[@]}" \
+  -o "$OUT" -Oz -fPIC -Wl,--no-entry -Wl,--strip-debug \
   -Wl,--export=malloc -Wl,--export=free \
   -Wl,--export=ts_parser_new -Wl,--export=ts_parser_delete \
   -Wl,--export=ts_parser_set_language -Wl,--export=ts_parser_parse_string \
-  -Wl,--export=ts_tree_root_node -Wl,--export=ts_tree_delete \
+  -Wl,--export=ts_parser_reset \
+  -Wl,--export=ts_tree_delete -Wl,--export=ts_tree_root_node \
   -Wl,--export=ts_node_child_count -Wl,--export=ts_node_child \
   -Wl,--export=ts_node_type -Wl,--export=ts_node_start_byte \
   -Wl,--export=ts_node_end_byte -Wl,--export=ts_node_has_error \
-  -Wl,--export=tree_sitter_typescript
+  -Wl,--export=ts_dump_tree -Wl,--export=ts_dump_rec_size \
+  -Wl,--export=ts_language_symbol_count -Wl,--export=ts_language_symbol_name \
+  "${EXPORTS[@]}"
 
-echo "built ts-ts.wasm ($(du -h ts-ts.wasm | cut -f1)) from tree-sitter $TS_VERSION + tree-sitter-typescript $TS_TS_VERSION"
+echo "built $OUT ($(du -h "$OUT" | cut -f1)) — runtime $TS_VERSION, ${#BUILT[@]} grammars"
+[ ${#FAILED[@]} -gt 0 ] && echo "FAILED: ${FAILED[*]}"
+echo "grammars: ${BUILT[*]}"

poc/wasm-treesitter/csrc/host_extra.c

@@ -0,0 +1,75 @@
+// host_extra.c — custom exports compiled INTO the wasm module alongside the
+// official tree-sitter runtime. The whole point is `ts_dump_tree`: it walks the
+// parsed tree ENTIRELY inside the guest and writes a flat pre-order array of
+// fixed-size records into linear memory in ONE shot. The host then does a single
+// Memory.Read and runs the chunker's node matching in pure Go — turning the
+// ~3 wazero calls-per-node of the naive walk into ~1 call per parse.
+//
+// See docs/wasm-treesitter-implementation-plan.md §7.2 / §7.3.
+#include "tree_sitter/api.h"
+#include <stdint.h>
+
+// One record per node. ALL fields uint32 so the Go side reads 9 little-endian
+// uint32s with zero packing ambiguity (recSize = 36). kind_id is the TSSymbol
+// (fits in 16 bits; widened for clean alignment). The host resolves kind_id ->
+// kind-name once per language via ts_language_symbol_name (see below), so the
+// per-node string lookup never crosses the boundary.
+typedef struct {
+  uint32_t kind_id;     // ts_node_symbol
+  uint32_t start_byte;
+  uint32_t end_byte;
+  uint32_t start_row;   // ts_node_start_point().row  (0-based line)
+  uint32_t start_col;   // ts_node_start_point().column
+  uint32_t end_row;     // ts_node_end_point().row
+  uint32_t end_col;     // ts_node_end_point().column
+  uint32_t depth;       // pre-order depth (parent reconstruction via depth stack)
+  uint32_t flags;       // bit0 named, bit1 error, bit2 missing, bit3 extra
+} NodeRec;
+
+static void emit(NodeRec *out, uint32_t i, TSNode n, uint32_t depth) {
+  out[i].kind_id    = ts_node_symbol(n);
+  out[i].start_byte = ts_node_start_byte(n);
+  out[i].end_byte   = ts_node_end_byte(n);
+  TSPoint sp = ts_node_start_point(n);
+  out[i].start_row  = sp.row;
+  out[i].start_col  = sp.column;
+  TSPoint ep = ts_node_end_point(n);
+  out[i].end_row    = ep.row;
+  out[i].end_col    = ep.column;
+  out[i].depth      = depth;
+  uint32_t f = 0;
+  if (ts_node_is_named(n))    f |= 1u;
+  if (ts_node_is_error(n))    f |= 2u;
+  if (ts_node_is_missing(n))  f |= 4u;
+  if (ts_node_is_extra(n))    f |= 8u;
+  out[i].flags = f;
+}
+
+// ts_dump_tree writes up to `cap` records and returns the TRUE node count. If the
+// return value > cap the buffer was too small: the host re-mallocs to the
+// returned count and calls again. Iterative pre-order DFS via a tree cursor — no
+// recursion, no per-node malloc, no boundary crossings.
+uint32_t ts_dump_tree(const TSTree *tree, NodeRec *out, uint32_t cap) {
+  if (tree == 0) return 0;
+  TSTreeCursor cur = ts_tree_cursor_new(ts_tree_root_node(tree));
+  uint32_t count = 0;
+  uint32_t depth = 0;
+  for (;;) {
+    if (count < cap) {
+      emit(out, count, ts_tree_cursor_current_node(&cur), depth);
+    }
+    count++;
+    if (ts_tree_cursor_goto_first_child(&cur)) { depth++; continue; }
+    for (;;) {
+      if (ts_tree_cursor_goto_next_sibling(&cur)) break;
+      if (!ts_tree_cursor_goto_parent(&cur)) {
+        ts_tree_cursor_delete(&cur);
+        return count;
+      }
+      depth--;
+    }
+  }
+}
+
+// recSize lets the host assert its struct layout matches the guest's.
+uint32_t ts_dump_rec_size(void) { return (uint32_t)sizeof(NodeRec); }

poc/wasm-treesitter/go.mod

@@ -2,7 +2,6 @@ module github.com/dvcdsys/code-index/poc/wasm-treesitter
 
 go 1.25.3
 
-require (
-	github.com/tetratelabs/wazero v1.12.0 // indirect
-	golang.org/x/sys v0.44.0 // indirect
-)
+require github.com/tetratelabs/wazero v1.12.0
+
+require golang.org/x/sys v0.44.0 // indirect

poc/wasm-treesitter/grammars_test.go

@@ -0,0 +1,75 @@
+package wasmts
+
+import (
+	"context"
+	"testing"
+)
+
+// minimal valid-ish snippet per base grammar — enough to exercise load + parse +
+// per-language symbol-table resolution through the batched path. We assert the
+// language loads, parsing does not trap, and produces nodes; per-language SYMBOL
+// correctness (which kinds = function/class/...) is the next phase.
+var smoke = []struct {
+	id  string
+	src string
+}{
+	{"python", "def f():\n    pass\n"},
+	{"typescript", "function f(){}\n"},
+	{"tsx", "const x = 1;\n"},
+	{"javascript", "function f(){}\n"},
+	{"go", "package m\nfunc F(){}\n"},
+	{"rust", "fn f(){}\n"},
+	{"java", "class C{}\n"},
+	{"c", "int f(){return 0;}\n"},
+	{"cpp", "int f(){return 0;}\n"},
+	{"ruby", "def f\nend\n"},
+	{"c_sharp", "class C{}\n"},
+	{"php", "<?php function f(){}\n"},
+	{"swift", "func f(){}\n"},
+	{"kotlin", "fun f(){}\n"},
+	{"scala", "object O{}\n"},
+	{"bash", "f(){ echo hi; }\n"},
+	{"lua", "function f() end\n"},
+	{"dart", "void f(){}\n"},
+	{"r", "f <- function() {}\n"},
+	{"objc", "int f(){return 0;}\n"},
+	{"html", "<div></div>\n"},
+	{"css", "a{color:red}\n"},
+	{"scss", "a{color:red}\n"},
+	{"sql", "SELECT 1;\n"},
+	{"markdown", "# Title\n"},
+	{"zig", "fn f() void {}\n"},
+	{"julia", "function f() end\n"},
+	{"fortran", "program p\nend program p\n"},
+	{"haskell", "main = return ()\n"},
+	{"ocaml", "let f () = ()\n"},
+	{"solidity", "contract C {}\n"},
+}
+
+func TestAllGrammarsLoad(t *testing.T) {
+	eng, err := New(context.Background(), 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer eng.Close()
+
+	if len(smoke) != 31 {
+		t.Fatalf("expected 31 grammars in smoke set, got %d", len(smoke))
+	}
+
+	for _, c := range smoke {
+		t.Run(c.id, func(t *testing.T) {
+			nodes, err := eng.ParseNodes("tree_sitter_"+c.id, []byte(c.src))
+			if err != nil {
+				t.Fatalf("parse trapped: %v", err)
+			}
+			if len(nodes) == 0 {
+				t.Fatal("no nodes produced")
+			}
+			// the root node must carry a resolved kind name (symbol table works)
+			if nodes[0].Kind == "" {
+				t.Errorf("root kind unresolved (symbol table empty?)")
+			}
+		})
+	}
+}