feat: serialize AST node back to wikitext string

core/modules/parsers/wikiparser/rules/commentblock.js

@@ -22,7 +22,7 @@ Note that the syntax for comments is simplified to an opening "<!--" sequence an
 "use strict";
 
 exports.name = "commentblock";
-exports.types = {block:true, pragma:true};
+exports.types = {block: true, pragma: true};
 
 exports.init = function(parser) {
 	this.parser = parser;
@@ -43,9 +43,18 @@ exports.findNextMatch = function(startPos) {
 	return undefined;
 };
 
+
 exports.parse = function() {
 	// Move past the match
 	this.parser.pos = this.endMatchRegExp.lastIndex;
-	// Don't return any elements
-	return [];
+	// Return a node representing the comment that is not rendered
+	var commentStart = this.match.index;
+	var commentEnd = this.endMatch.index + this.endMatch[0].length;
+	return [{
+			type: "void",
+			children: [],
+			text: this.parser.source.slice(commentStart, commentEnd),
+			start: commentStart,
+			end: commentEnd
+	}];
 };

core/modules/parsers/wikiparser/rules/commentinline.js

@@ -40,6 +40,13 @@ exports.findNextMatch = function(startPos) {
 exports.parse = function() {
 	// Move past the match
 	this.parser.pos = this.endMatchRegExp.lastIndex;
-	// Don't return any elements
-	return [];
+	// Return a node representing the inline comment
+	var commentStart = this.match.index;
+	var commentEnd = this.endMatch.index + this.endMatch[0].length;
+	return [{
+			type: "void",
+			text: this.parser.source.slice(commentStart, commentEnd),
+			start: commentStart,
+			end: commentEnd
+	}];
 };

core/modules/parsers/wikiparser/rules/extlink.js

@@ -28,11 +28,11 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	// Move past the match
-    var start = this.parser.pos;
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Create the link unless it is suppressed
 	if(this.match[0].substr(0,1) === "~") {
-		return [{type: "text", text: this.match[0].substr(1)}];
+		return [{type: "text", text: this.match[0].substr(1), start: start, end: this.parser.pos}];
 	} else {
 		return [{
 			type: "element",

core/modules/parsers/wikiparser/rules/fnprocdef.js

@@ -6,15 +6,15 @@ module-type: wikirule
 Wiki pragma rule for function, procedure and widget definitions
 
 ```
-\function name(param:defaultvalue,param2:defaultvalue)
+\function name(param:"defaultvalue", param2:"defaultvalue")
 definition text
 \end
 
-\procedure name(param:defaultvalue,param2:defaultvalue)
+\procedure name(param:"defaultvalue", param2:"defaultvalue")
 definition text
 \end
 
-\widget $mywidget(param:defaultvalue,param2:defaultvalue)
+\widget $mywidget(param:"defaultvalue", param2:"defaultvalue")
 definition text
 \end
 ```

core/modules/parsers/wikiparser/rules/hardlinebreaks.js

@@ -50,6 +50,8 @@ exports.parse = function() {
 			}
 		}
 	} while(match && !match[1]);
-	// Return the nodes
+	// Mark first and last node, and return the nodes
+	if(tree[0]) tree[0].isRuleStart = true;
+	if(tree[tree.length-1]) tree[tree.length-1].isRuleEnd = true;
 	return tree;
 };

core/modules/parsers/wikiparser/rules/html.js

@@ -41,7 +41,7 @@ Parse the most recent match
 exports.parse = function() {
 	// Retrieve the most recent match so that recursive calls don't overwrite it
 	var tag = this.nextTag;
-	if (!tag.isSelfClosing) {
+	if(!tag.isSelfClosing) {
 		tag.openTagStart = tag.start;
 		tag.openTagEnd = tag.end;
 	}
@@ -63,22 +63,22 @@ exports.parse = function() {
 		}
 		tag.end = this.parser.pos;
 		tag.closeTagEnd = tag.end;
-		if (tag.closeTagEnd === tag.openTagEnd || this.parser.source[tag.closeTagEnd - 1] !== '>') {
+		if(tag.closeTagEnd === tag.openTagEnd || this.parser.source[tag.closeTagEnd - 1] !== ">") {
 			tag.closeTagStart = tag.end;
 		} else {
 			tag.closeTagStart = tag.closeTagEnd - 2;
 			var closeTagMinPos = tag.children.length > 0 ? tag.children[tag.children.length-1].end : tag.openTagEnd;
-			if (!Number.isSafeInteger(closeTagMinPos)) closeTagMinPos = tag.openTagEnd;
-			while (tag.closeTagStart >= closeTagMinPos) {
+			if(!Number.isSafeInteger(closeTagMinPos)) closeTagMinPos = tag.openTagEnd;
+			while(tag.closeTagStart >= closeTagMinPos) {
 				var char = this.parser.source[tag.closeTagStart];
-				if (char === '>') {
+				if(char === ">") {
 					tag.closeTagStart = -1;
 					break;
 				}
-				if (char === '<') break;
+				if(char === "<") break;
 				tag.closeTagStart -= 1;
 			}
-			if (tag.closeTagStart < closeTagMinPos) {
+			if(tag.closeTagStart < closeTagMinPos) {
 				tag.closeTagStart = tag.end;
 			}
 		}

core/modules/parsers/wikiparser/rules/list.js

@@ -59,6 +59,7 @@ var listTypes = {
 	":": {listTag: "dl", itemTag: "dd"},
 	">": {listTag: "blockquote", itemTag: "div"}
 };
+exports.listTypes = listTypes;
 
 /*
 Parse the most recent match

core/modules/parsers/wikiparser/rules/macrocallblock.js

@@ -29,7 +29,7 @@ exports.findNextMatch = function(startPos) {
 			var c = this.parser.source.charAt(nextCall.end);
 			// Ensure EOL after parsed macro
 			// If we didn't need to support IE, we'd just use /(?:\r?\n|$)/ym
-			if ((c === "") || (c === "\n") || ((c === "\r") && this.parser.source.charAt(nextCall.end+1) === "\n")) {
+			if((c === "") || (c === "\n") || ((c === "\r") && this.parser.source.charAt(nextCall.end+1) === "\n")) {
 				this.nextCall = nextCall;
 				return nextStart;
 			}

core/modules/parsers/wikiparser/rules/macrocallinline.js

@@ -42,3 +42,5 @@ exports.parse = function() {
 	this.parser.pos = call.end;
 	return [call];
 };
+
+

core/modules/parsers/wikiparser/rules/macrodef.js

@@ -52,9 +52,10 @@ exports.parse = function() {
 		}
 	}
 	// Is the remainder of the \define line blank after the parameter close paren?
-	var reEnd;
+	var reEnd,isBlock = true;
 	if(this.match[3]) {
 		// If so, it is a multiline definition and the end of the body is marked with \end
+		isBlock = false;
 		reEnd = new RegExp("((?:^|\\r?\\n)[^\\S\\n\\r]*\\\\end[^\\S\\n\\r]*(?:" + $tw.utils.escapeRegExp(this.match[1]) + ")?\\s*?(?:$|\\r?\\n))","mg");
 	} else {
 		// Otherwise, the end of the definition is marked by the end of the line
@@ -79,7 +80,8 @@ exports.parse = function() {
 		attributes: {},
 		children: [],
 		params: params,
-		isMacroDefinition: true
+		isMacroDefinition: true,
+		isBlock: isBlock && !!endMatch
 	}];
 	$tw.utils.addAttributeToParseTreeNode(parseTreeNodes[0],"name",this.match[1]);
 	$tw.utils.addAttributeToParseTreeNode(parseTreeNodes[0],"value",text);

core/modules/parsers/wikiparser/rules/parsermode.js

@@ -31,6 +31,7 @@ Parse the most recent match
 */
 exports.parse = function() {
 	// Move past the pragma invocation
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Parse whitespace delimited tokens terminated by a line break
 	var reMatch = /[^\S\n]*(\S+)|(\r?\n)/mg,
@@ -58,6 +59,11 @@ exports.parse = function() {
 			this.parser.parseAsInline = true;
 		}
 	}
-	// No parse tree nodes to return
-	return [];
+	return [{
+		type: "void",
+		children: [],
+		parseAsInline: this.parser.parseAsInline,
+		start: start,
+		end: this.parser.pos
+	}];
 };

core/modules/parsers/wikiparser/rules/prettyextlink.js

@@ -113,3 +113,5 @@ exports.parseLink = function(source,pos) {
 	node.end = closePos + 2;
 	return node;
 };
+
+

core/modules/parsers/wikiparser/rules/prettylink.js

@@ -32,7 +32,7 @@ exports.parse = function() {
 	var text = this.match[1],
 		link = this.match[2] || text,
 		textEndPos = this.parser.source.indexOf("|", start);
-	if (textEndPos < 0 || textEndPos > this.matchRegExp.lastIndex) {
+	if(textEndPos < 0 || textEndPos > this.matchRegExp.lastIndex) {
 		textEndPos = this.matchRegExp.lastIndex - 2;
 	}
 	var linkStart = this.match[2] ? (start + this.match[1].length + 1) : start;

core/modules/parsers/wikiparser/rules/rules.js

@@ -54,6 +54,13 @@ exports.parse = function() {
 	if(tokens.length > 0) {
 		this.parser.amendRules(tokens[0],tokens.slice(1));
 	}
-	// No parse tree nodes to return
-	return [];
+	// No widget to render, return void node.
+	return [{
+		type: "void",
+		attributes: {
+			action: {type: "string", value: tokens[0]},
+			rules: {type: "string", value: tokens.slice(1).join(" ")}
+		},
+		children: []
+	}];
 };

core/modules/parsers/wikiparser/rules/styleblock.js

@@ -64,5 +64,8 @@ exports.parse = function() {
 			$tw.utils.addAttributeToParseTreeNode(tree[t],"style",styles.join(""));
 		}
 	}
-	return tree;
+	return [{
+		type: "void",
+		children: tree
+	}]
 };

core/modules/parsers/wikiparser/rules/styleinline.js

@@ -21,7 +21,7 @@ exports.types = {inline: true};
 
 exports.init = function(parser) {
 	this.parser = parser;
-	// Regexp to match
+	// Regexp to match /@@(styles)?\s*(\.class\s+)?/
 	this.matchRegExp = /@@((?:[^\.\r\n\s:]+:[^\r\n;]+;)+)?(\.(?:[^\r\n\s]+)\s+)?/mg;
 };
 

core/modules/parsers/wikiparser/rules/typedblock.js

@@ -60,22 +60,37 @@ exports.parse = function() {
 	var parser = this.parser.wiki.parseText(parseType,text,{defaultType: "text/plain"});
 	// If there's no render type, just return the parse tree
 	if(!renderType) {
-		return parser.tree;
+		return  [{
+			type: "void",
+			children: $tw.utils.isArray(parser.tree) ? parser.tree : [parser.tree],
+			parseType: parseType,
+			renderType: renderType,
+			text: text,
+			start: start,
+			end: this.parser.pos
+		}];
 	} else {
 		// Otherwise, render to the rendertype and return in a <PRE> tag
 		var widgetNode = this.parser.wiki.makeWidget(parser),
 			container = $tw.fakeDocument.createElement("div");
 		widgetNode.render(container,null);
-		text = renderType === "text/html" ? container.innerHTML : container.textContent;
+		var renderResult = renderType === "text/html" ? container.innerHTML : container.textContent;
+		// Use void node to carry important info for typedblock
 		return [{
-			type: "element",
-			tag: "pre",
+			type: "void",
 			children: [{
-				type: "text",
-				text: text,
-				start: start,
-				end: this.parser.pos
-			}]
+				type: "element",
+				tag: "pre",
+				children: [{
+					type: "text",
+					text: renderResult,
+				}]
+			}],
+			parseType: parseType,
+			renderType: renderType,
+			text: text,
+			start: start,
+			end: this.parser.pos
 		}];
 	}
 };

core/modules/parsers/wikiparser/wikiparser.js

@@ -215,8 +215,8 @@ WikiParser.prototype.parsePragmas = function() {
 		var subTree = nextMatch.rule.parse();
 		if(subTree.length > 0) {
 			// Set the start and end positions of the pragma rule if
-			if (subTree[0].start === undefined) subTree[0].start = start;
-			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			if(subTree[0].start === undefined) subTree[0].start = start;
+			if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 			$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
 			// Quick hack; we only cope with a single parse tree node being returned, which is true at the moment
 			currentTreeBranch.push.apply(currentTreeBranch,subTree);
@@ -245,9 +245,9 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
 		var start = this.pos;
 		var subTree = nextMatch.rule.parse();
 		// Set the start and end positions of the first and last blocks if they're not already set
-		if (subTree.length > 0) {
-			if (subTree[0].start === undefined) subTree[0].start = start;
-			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+		if(subTree.length > 0) {
+			if(subTree[0].start === undefined) subTree[0].start = start;
+			if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 		}
 		$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
 		return subTree;
@@ -256,7 +256,7 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
 	var start = this.pos;
 	var children = this.parseInlineRun(terminatorRegExp);
 	var end = this.pos;
-	return [{type: "element", tag: "p", children: children, start: start, end: end }];
+	return [{type: "element", tag: "p", children: children, start: start, end: end, rule: "parseblock" }];
 };
 
 /*
@@ -350,10 +350,10 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) {
 		var start = this.pos;
 		var subTree = nextMatch.rule.parse();
 		// Set the start and end positions of the first and last child if they're not already set
-		if (subTree.length > 0) {
+		if(subTree.length > 0) {
 			// Set the start and end positions of the first and last child if they're not already set
-			if (subTree[0].start === undefined) subTree[0].start = start;
-			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			if(subTree[0].start === undefined) subTree[0].start = start;
+			if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 		}
 		$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
 		tree.push.apply(tree,subTree);
@@ -410,9 +410,9 @@ WikiParser.prototype.parseInlineRunTerminatedExtended = function(terminatorRegEx
 			var start = this.pos;
 			var subTree = inlineRuleMatch.rule.parse();
 			// Set the start and end positions of the first and last child if they're not already set
-			if (subTree.length > 0) {
-				if (subTree[0].start === undefined) subTree[0].start = start;
-				if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			if(subTree.length > 0) {
+				if(subTree[0].start === undefined) subTree[0].start = start;
+				if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 			}
 			$tw.utils.each(subTree, function (node) { node.rule = inlineRuleMatch.rule.name; });
 			tree.push.apply(tree,subTree);

core/modules/utils/parsetree.js

@@ -119,3 +119,19 @@ exports.getParseTreeText = function getParseTreeText(tree) {
 	}
 	return output.join("");
 };
+
+exports.getParser = function(type,options) {
+	options = options || {};
+	// Select a parser
+	var Parser = $tw.Wiki.parsers[type];
+	if(!Parser && $tw.utils.getFileExtensionInfo(type)) {
+		Parser = $tw.Wiki.parsers[$tw.utils.getFileExtensionInfo(type).type];
+	}
+	if(!Parser) {
+		Parser = $tw.Wiki.parsers[options.defaultType || "text/vnd.tiddlywiki"];
+	}
+	if(!Parser) {
+		return null;
+	}
+	return Parser;
+};

core/modules/widgets/importvariables.js

@@ -49,7 +49,8 @@ ImportVariablesWidget.prototype.execute = function(tiddlerList) {
 		var parser = widgetPointer.wiki.parseTiddler(title,{parseAsInline:true, configTrimWhiteSpace:false});
 		if(parser) {
 			var parseTreeNode = parser.tree[0];
-			while(parseTreeNode && ["setvariable","set","parameters"].indexOf(parseTreeNode.type) !== -1) {
+			// process AST nodes generated by pragma rules.
+			while(parseTreeNode && ["setvariable","set","parameters","void"].indexOf(parseTreeNode.type) !== -1) {
 				var node = {
 					type: "set",
 					attributes: parseTreeNode.attributes,
@@ -82,7 +83,7 @@ ImportVariablesWidget.prototype.execute = function(tiddlerList) {
 						// this widget. If it needs to refresh,
 						// it'll do so along with the the whole
 						// importvariable tree.
-						if (widgetPointer != this) {
+						if(widgetPointer != this) {
 							widgetPointer.makeChildWidgets = function(){};
 						}
 						widgetPointer = widgetPointer.children[0];
@@ -93,7 +94,7 @@ ImportVariablesWidget.prototype.execute = function(tiddlerList) {
 		} 
 	});
 
-	if (widgetPointer != this) {
+	if(widgetPointer != this) {
 		widgetPointer.parseTreeNode.children = this.parseTreeNode.children;
 	} else {
 		widgetPointer.makeChildWidgets();