Aho-Corasick Freelinks Enhancement for Large Wikis and Non-Latin Titles

plugins/tiddlywiki/freelinks/text.js

@@ -3,19 +3,136 @@ title: $:/core/modules/widgets/text.js
 type: application/javascript
 module-type: widget
 
-An override of the core text widget that automatically linkifies the text
+An override of the core text widget that automatically linkifies the text, with support for non-Latin languages like Chinese, prioritizing longer titles, skipping processed matches, excluding the current tiddler title from linking, and handling large title sets with Aho-Corasick algorithm and fixed chunking (100 titles per chunk). Includes optional persistent caching of Aho-Corasick automaton, controlled by $:/config/Freelinks/PersistAhoCorasickCache.
 
 \*/
+(function(){
 
+/*jslint node: true, browser: true */
+/*global $tw: false */
 "use strict";
 
 var TITLE_TARGET_FILTER = "$:/config/Freelinks/TargetFilter";
+var PERSIST_CACHE_TIDDLER = "$:/config/Freelinks/PersistAhoCorasickCache";
 
 var Widget = require("$:/core/modules/widgets/widget.js").widget,
 	LinkWidget = require("$:/core/modules/widgets/link.js").link,
 	ButtonWidget = require("$:/core/modules/widgets/button.js").button,
 	ElementWidget = require("$:/core/modules/widgets/element.js").element;
 
+// Escape only ASCII 127 and below metacharacters
+function escapeRegExp(str) {
+	try {
+		return str.replace(/[\\^$*+?.()|[\]{}]/g, '\\$&');
+	} catch(e) {
+		console.warn("Failed to escape title:", str, e);
+		return null; // Skip problematic titles
+	}
+}
+
+// Aho-Corasick implementation with enhanced error handling
+function AhoCorasick() {
+	this.trie = {};
+	this.failure = {};
+	this.output = {};
+}
+
+AhoCorasick.prototype.addPattern = function(pattern, index) {
+	if(!pattern || typeof pattern !== "string") {
+		console.warn("Skipping invalid pattern:", pattern);
+		return;
+	}
+	var node = this.trie;
+	for(var i = 0; i < pattern.length; i++) {
+		var char = pattern[i];
+		if(!node[char]) {
+			node[char] = {};
+		}
+		node = node[char];
+	}
+	if(!node.$) {
+		node.$ = [];
+	}
+	node.$.push({ pattern: pattern, index: index });
+};
+
+AhoCorasick.prototype.buildFailureLinks = function() {
+	var queue = [];
+	var root = this.trie;
+	this.failure[root] = root;
+	for(var char in root) {
+		if(root[char] && char !== '$') {
+			this.failure[root[char]] = root;
+			queue.push(root[char]);
+		}
+	}
+	var maxIterations = 100000; // Prevent infinite loops
+	var iteration = 0;
+	while(queue.length && iteration < maxIterations) {
+		var node = queue.shift();
+		for(var char in node) {
+			if(node[char] && char !== '$') {
+				var child = node[char];
+				var fail = this.failure[node];
+				var failCount = 0;
+				var maxFailCount = 1000; // Prevent deep failure chains
+				while(fail && !fail[char] && failCount < maxFailCount) {
+					fail = this.failure[fail];
+					failCount++;
+				}
+				this.failure[child] = fail[char] || this.trie;
+				if(this.failure[child].$) {
+					if(!child.$) {
+						child.$ = [];
+					}
+					child.$.push.apply(child.$, this.failure[child].$);
+				}
+				queue.push(child);
+			}
+		}
+		iteration++;
+	}
+	if(iteration >= maxIterations) {
+		console.error("Aho-Corasick: buildFailureLinks exceeded max iterations");
+	}
+};
+
+AhoCorasick.prototype.search = function(text) {
+	if(!text || typeof text !== "string") {
+		return [];
+	}
+	var matches = [];
+	var node = this.trie;
+	var maxIterations = text.length * 10; // Prevent infinite loops
+	var iteration = 0;
+	for(var i = 0; i < text.length && iteration < maxIterations; i++) {
+		var char = text[i];
+		var transitionCount = 0;
+		var maxTransitionCount = 1000; // Prevent deep transitions
+		while(node && !node[char] && transitionCount < maxTransitionCount) {
+			node = this.failure[node];
+			transitionCount++;
+		}
+		node = node[char] || this.trie;
+		if(node.$) {
+			for(var j = 0; j < node.$.length; j++) {
+				var match = node.$[j];
+				matches.push({
+					pattern: match.pattern,
+					index: i - match.pattern.length + 1,
+					length: match.pattern.length,
+					titleIndex: match.index
+				});
+			}
+		}
+		iteration++;
+	}
+	if(iteration >= maxIterations) {
+		console.error("Aho-Corasick: search exceeded max iterations");
+	}
+	return matches;
+};
+
 var TextNodeWidget = function(parseTreeNode,options) {
 	this.initialise(parseTreeNode,options);
 };
@@ -43,91 +160,170 @@ TextNodeWidget.prototype.execute = function() {
 		ignoreCase = self.getVariable("tv-freelinks-ignore-case",{defaultValue:"no"}).trim() === "yes";
 	// Get our parameters
 	var childParseTree = [{
-			type: "plain-text",
-			text: this.getAttribute("text",this.parseTreeNode.text || "")
-		}];
+		type: "plain-text",
+		text: this.getAttribute("text",this.parseTreeNode.text || "")
+	}];
 	// Only process links if not disabled and we're not within a button or link widget
 	if(this.getVariable("tv-wikilinks",{defaultValue:"yes"}).trim() !== "no" && this.getVariable("tv-freelinks",{defaultValue:"no"}).trim() === "yes" && !this.isWithinButtonOrLink()) {
-		// Get the information about the current tiddler titles, and construct a regexp
-		this.tiddlerTitleInfo = this.wiki.getGlobalCache("tiddler-title-info-" + (ignoreCase ? "insensitive" : "sensitive"),function() {
-			var targetFilterText = self.wiki.getTiddlerText(TITLE_TARGET_FILTER),
-				titles = !!targetFilterText ? self.wiki.filterTiddlers(targetFilterText,$tw.rootWidget) : self.wiki.allTitles(),
-				sortedTitles = titles.sort(function(a,b) {
-					var lenA = a.length,
-						lenB = b.length;
-					// First sort by length, so longer titles are first
-					if(lenA !== lenB) {
-						if(lenA < lenB) {
-							return +1;
-						} else {
-							return -1;
-						}
-					} else {
-					// Then sort alphabetically within titles of the same length
-						if(a < b) {
-							return -1;
-						} else if(a > b) {
-							return +1;
-						} else {
-							return 0;
-						}
-					}
-				}),
-				titles = [],
-				reparts = [];
-			$tw.utils.each(sortedTitles,function(title) {
-				if(title.substring(0,3) !== "$:/") {
-					titles.push(title);
-					reparts.push("(" + $tw.utils.escapeRegExp(title) + ")");
-				}
+		// Get the current tiddler title
+		var currentTiddlerTitle = this.getVariable("currentTiddler") || "";
+		// Check if persistent caching is enabled
+		var persistCache = self.wiki.getTiddlerText(PERSIST_CACHE_TIDDLER, "no").trim() === "yes";
+		var cacheKey = "tiddler-title-info-" + (ignoreCase ? "insensitive" : "sensitive");
+		// Get the information about the current tiddler titles
+		this.tiddlerTitleInfo = persistCache ?
+			this.wiki.getPersistentCache(cacheKey, function() {
+				return computeTiddlerTitleInfo(self, ignoreCase);
+			}) :
+			this.wiki.getGlobalCache(cacheKey, function() {
+				return computeTiddlerTitleInfo(self, ignoreCase);
 			});
-			var regexpStr = "\\b(?:" + reparts.join("|") + ")\\b";
-			return {
-				titles: titles,
-				regexp: new RegExp(regexpStr,ignoreCase ? "i" : "")
-			};
-		});
-		// Repeatedly linkify
+		// Process titles to avoid overlapping matches
 		if(this.tiddlerTitleInfo.titles.length > 0) {
-			var index,text,match,matchEnd;
-			do {
-				index = childParseTree.length - 1;
-				text = childParseTree[index].text;
-				match = this.tiddlerTitleInfo.regexp.exec(text);
-				if(match) {
-					// Make a text node for any text before the match
-					if(match.index > 0) {
-						childParseTree[index].text = text.substring(0,match.index);
-						index += 1;
+			var text = childParseTree[0].text,
+				newParseTree = [],
+				currentPos = 0;
+			// Use Aho-Corasick to find all matches
+			var searchText = ignoreCase ? text.toLowerCase() : text;
+			var matches;
+			try {
+				matches = this.tiddlerTitleInfo.ac.search(searchText);
+			} catch(e) {
+				console.error("Aho-Corasick search failed:", e);
+				matches = [];
+			}
+			// Sort matches by position and length (longer titles first)
+			matches.sort(function(a, b) {
+				if(a.index !== b.index) {
+					return a.index - b.index;
+				}
+				return b.length - a.length;
+			});
+			// Process matches to avoid overlaps
+			var processedPositions = new Set();
+			for(var i = 0; i < matches.length; i++) {
+				var match = matches[i];
+				var matchStart = match.index;
+				var matchEnd = matchStart + match.length;
+				// Skip if position already processed (ensures longer titles take precedence)
+				var overlap = false;
+				for(var pos = matchStart; pos < matchEnd; pos++) {
+					if(processedPositions.has(pos)) {
+						overlap = true;
+						break;
 					}
-					// Make a link node for the match
-					childParseTree[index] = {
+				}
+				if(overlap) {
+					continue;
+				}
+				// Mark positions as processed
+				for(var pos = matchStart; pos < matchEnd; pos++) {
+					processedPositions.add(pos);
+				}
+				// Add text before the match
+				if(matchStart > currentPos) {
+					newParseTree.push({
+						type: "plain-text",
+						text: text.slice(currentPos, matchStart)
+					});
+				}
+				// Get matched title
+				var matchedTitle = this.tiddlerTitleInfo.titles[match.titleIndex];
+				// Check if the matched title is the current tiddler title
+				if(matchedTitle === currentTiddlerTitle) {
+					// Skip linking, keep as plain text
+					newParseTree.push({
+						type: "plain-text",
+						text: text.slice(matchStart, matchEnd)
+					});
+				} else {
+					// Add link for the match
+					newParseTree.push({
 						type: "link",
 						attributes: {
-							to: {type: "string", value: ignoreCase ? this.tiddlerTitleInfo.titles[match.indexOf(match[0],1) - 1] : match[0]},
+							to: {type: "string", value: matchedTitle},
 							"class": {type: "string", value: "tc-freelink"}
 						},
 						children: [{
-							type: "plain-text", text: match[0]
-						}]
-					};
-					index += 1;
-					// Make a text node for any text after the match
-					matchEnd = match.index + match[0].length;
-					if(matchEnd < text.length) {
-						childParseTree[index] = {
 							type: "plain-text",
-							text: text.substring(matchEnd)
-						};					
-					}
+							text: text.slice(matchStart, matchEnd)
+						}]
+					});
 				}
-			} while(match && childParseTree[childParseTree.length - 1].type === "plain-text");			
+				currentPos = matchEnd;
+			}
+			// Add remaining text
+			if(currentPos < text.length) {
+				newParseTree.push({
+					type: "plain-text",
+					text: text.slice(currentPos)
+				});
+			}
+			childParseTree = newParseTree;
 		}
 	}
 	// Make the child widgets
 	this.makeChildWidgets(childParseTree);
 };
 
+/*
+Helper function to compute tiddler title info
+*/
+function computeTiddlerTitleInfo(self, ignoreCase) {
+	var targetFilterText = self.wiki.getTiddlerText(TITLE_TARGET_FILTER),
+		titles = !!targetFilterText ? self.wiki.filterTiddlers(targetFilterText,$tw.rootWidget) : self.wiki.allTitles();
+	if(!titles || titles.length === 0) {
+		console.warn("No titles found for Freelinks processing");
+		return { titles: [], ac: new AhoCorasick(), chunkSize: 100, titleChunks: [] };
+	}
+	var sortedTitles = titles.sort(function(a,b) {
+			var lenA = a.length,
+				lenB = b.length;
+			// Sort by length (longer first), then alphabetically for same length
+			if(lenA !== lenB) {
+				return lenB - lenA;
+			} else {
+				return a.localeCompare(b, 'zh', {sensitivity: 'base'});
+			}
+		}),
+		titles = [],
+		chunkSize = 100; // Fixed chunk size
+	var ac = new AhoCorasick();
+	$tw.utils.each(sortedTitles,function(title, index) {
+		if(title.substring(0,3) !== "$:/") {
+			var escapedTitle = escapeRegExp(title);
+			if(escapedTitle) {
+				titles.push(title);
+				ac.addPattern(ignoreCase ? title.toLowerCase() : title, titles.length - 1);
+			} else {
+				console.warn("Skipping invalid title:", title);
+			}
+		}
+	});
+	try {
+		ac.buildFailureLinks();
+	} catch(e) {
+		console.error("Failed to build Aho-Corasick failure links:", e);
+		return { titles: [], ac: new AhoCorasick(), chunkSize: 100, titleChunks: [] };
+	}
+	// Split titles into chunks for memory management
+	var titleChunks = [];
+	for(var i = 0; i < titles.length; i += chunkSize) {
+		titleChunks.push(titles.slice(i, i + chunkSize));
+	}
+	// Log chunking details for debugging
+	console.log("Total titles:", titles.length, ", Chunk size:", chunkSize, ", Number of chunks:", titleChunks.length, ", Persistent cache:", self.wiki.getTiddlerText(PERSIST_CACHE_TIDDLER, "no").trim());
+	return {
+		titles: titles,
+		ac: ac,
+		chunkSize: chunkSize,
+		titleChunks: titleChunks
+	};
+}
+
+/*
+Check if the widget is within a button or link
+*/
 TextNodeWidget.prototype.isWithinButtonOrLink = function() {
 	var withinButtonOrLink = false,
 		widget = this.parentWidget;
@@ -153,11 +349,20 @@ TextNodeWidget.prototype.refresh = function(changedTiddlers) {
 		}
 	});
 	if(changedAttributes.text || titlesHaveChanged) {
+		// Invalidate cache if titles changed and persistent cache is enabled
+		var persistCache = self.wiki.getTiddlerText(PERSIST_CACHE_TIDDLER, "no").trim() === "yes";
+		var cacheKey = "tiddler-title-info-" + (self.getVariable("tv-freelinks-ignore-case",{defaultValue:"no"}).trim() === "yes" ? "insensitive" : "sensitive");
+		if(titlesHaveChanged && persistCache) {
+			self.wiki.clearCache(cacheKey);
+			console.log("Cleared Aho-Corasick cache due to title changes");
+		}
 		this.refreshSelf();
 		return true;
 	} else {
-		return false;	
+		return false;
 	}
 };
 
 exports.text = TextNodeWidget;
+
+})();