diff --git a/.gitignore b/.gitignore index 74ab03195..a1c22e42c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +npm-debug.log .DS_Store latest-change.txt patternlab.json diff --git a/core/lib/list_item_hunter.js b/core/lib/list_item_hunter.js index 838851aec..cb1eadd63 100644 --- a/core/lib/list_item_hunter.js +++ b/core/lib/list_item_hunter.js @@ -13,6 +13,7 @@ var list_item_hunter = function () { var extend = require('util')._extend, + JSON5 = require('json5'), pa = require('./pattern_assembler'), smh = require('./style_modifier_hunter'), pattern_assembler = new pa(), @@ -44,7 +45,13 @@ var list_item_hunter = function () { } //check for a local listitems.json file - var listData = JSON.parse(JSON.stringify(patternlab.listitems)); + var listData; + try { + listData = JSON5.parse(JSON5.stringify(patternlab.listitems)); + } catch (err) { + console.log('There was an error parsing JSON for ' + pattern.abspath); + console.log(err); + } listData = pattern_assembler.merge_data(listData, pattern.listitems); //iterate over each copied block, rendering its contents along with pattenlab.listitems[i] @@ -54,8 +61,15 @@ var list_item_hunter = function () { //combine listItem data with pattern data with global data var itemData = listData['' + items.indexOf(loopNumberString)]; //this is a property like "2" - var globalData = JSON.parse(JSON.stringify(patternlab.data)); - var localData = JSON.parse(JSON.stringify(pattern.jsonFileData)); + var globalData; + var localData; + try { + globalData = JSON5.parse(JSON5.stringify(patternlab.data)); + localData = JSON5.parse(JSON5.stringify(pattern.jsonFileData)); + } catch (err) { + console.log('There was an error parsing JSON for ' + pattern.abspath); + console.log(err); + } var allData = pattern_assembler.merge_data(globalData, localData); allData = pattern_assembler.merge_data(allData, itemData !== undefined ? itemData[i] : {}); //itemData could be undefined if the listblock contains no partial, just markup @@ -71,7 +85,13 @@ var list_item_hunter = function () { var partialPattern = pattern_assembler.get_pattern_by_key(partialName, patternlab); //create a copy of the partial so as to not pollute it after the get_pattern_by_key call. - var cleanPartialPattern = JSON.parse(JSON.stringify(partialPattern)); + var cleanPartialPattern; + try { + cleanPartialPattern = JSON5.parse(JSON5.stringify(partialPattern)); + } catch (err) { + console.log('There was an error parsing JSON for ' + pattern.abspath); + console.log(err); + } //if partial has style modifier data, replace the styleModifier value if (foundPartials[j].indexOf(':') > -1) { diff --git a/core/lib/parameter_hunter.js b/core/lib/parameter_hunter.js index f9f7cfca5..e6adef692 100644 --- a/core/lib/parameter_hunter.js +++ b/core/lib/parameter_hunter.js @@ -13,110 +13,230 @@ var parameter_hunter = function () { var extend = require('util')._extend, + JSON5 = require('json5'), pa = require('./pattern_assembler'), smh = require('./style_modifier_hunter'), - style_modifier_hunter = new smh(), - pattern_assembler = new pa(); - + pattern_assembler = new pa(), + style_modifier_hunter = new smh(); + + /** + * This function is really to accommodate the lax JSON-like syntax allowed by + * Pattern Lab PHP for parameter submissions to partials. Unfortunately, no + * easily searchable library was discovered for this. What we had to do was + * write a custom script to crawl through the parameter string, and wrap the + * keys and values in double-quotes as necessary. + * The steps on a high-level are as follows: + * * Further escape all escaped quotes and colons. Use the string + * representation of their unicodes for this. This has the added bonus + * of being interpreted correctly by JSON5.parse() without further + * modification. This will be useful later in the function. + * * Once escaped quotes are out of the way, we know the remaining quotes + * are either key/value wrappers or wrapped within those wrappers. We know + * that remaining commas and colons are either delimiters, or wrapped + * within quotes to not be recognized as such. + * * A do-while loop crawls paramString to write keys to a keys array and + * values to a values array. + * * Start by parsing the first key. Determine the type of wrapping quote, + * if any. + * * By knowing the open wrapper, we know that the next quote of that kind + * (if the key is wrapped in quotes), HAS to be the close wrapper. + * Similarly, if the key is unwrapped, we know the next colon HAS to be + * the delimiter between key and value. + * * Save the key to the keys array. + * * Next, search for a value. It will either be the next block wrapped in + * quotes, or a string of alphanumerics, decimal points, or minus signs. + * * Save the value to the values array. + * * The do-while loop truncates the paramString value while parsing. Its + * condition for completion is when the paramString is whittled down to an + * empty string. + * * After the keys and values arrays are built, a for loop iterates through + * them to build the final paramStringWellFormed string. + * * No quote substitution had been done prior to this loop. In this loop, + * all keys are ensured to be wrapped in double-quotes. String values are + * also ensured to be wrapped in double-quotes. + * * Unescape escaped unicodes except for double-quotes. Everything beside + * double-quotes will be wrapped in double-quotes without need for escape. + * * Return paramStringWellFormed. + * + * @param {string} pString + * @returns {string} paramStringWellFormed + */ function paramToJson(pString) { - var paramStringWellFormed = ''; - var paramStringTmp; - var colonPos; - var delimitPos; - var quotePos; - var paramString = pString; - + var colonPos = -1; + var keys = []; + var paramString = pString; // to not reassign param + var paramStringWellFormed; + var quotePos = -1; + var regex; + var values = []; + var wrapper; + + //replace all escaped double-quotes with escaped unicode + paramString = paramString.replace(/\\"/g, '\\u0022'); + + //replace all escaped single-quotes with escaped unicode + paramString = paramString.replace(/\\'/g, '\\u0027'); + + //replace all escaped colons with escaped unicode + paramString = paramString.replace(/\\:/g, '\\u0058'); + + //with escaped chars out of the way, crawl through paramString looking for + //keys and values do { - //if param key is wrapped in single quotes, replace with double quotes. - paramString = paramString.replace(/(^\s*[\{|\,]\s*)'([^']+)'(\s*\:)/, '$1"$2"$3'); + //check if searching for a key + if (paramString[0] === '{' || paramString[0] === ',') { + paramString = paramString.substring(1, paramString.length).trim(); - //if params key is not wrapped in any quotes, wrap in double quotes. - paramString = paramString.replace(/(^\s*[\{|\,]\s*)([^\s"'\:]+)(\s*\:)/, '$1"$2"$3'); + //search for end quote if wrapped in quotes. else search for colon. + //everything up to that position will be saved in the keys array. + switch (paramString[0]) { - //move param key to paramStringWellFormed var. - colonPos = paramString.indexOf(':'); + //need to search for end quote pos in case the quotes wrap a colon + case '"': + case '\'': + wrapper = paramString[0]; + quotePos = paramString.indexOf(wrapper, 1); + break; - //except to prevent infinite loops. - if (colonPos === -1) { - colonPos = paramString.length - 1; - } - else { - colonPos += 1; - } - paramStringWellFormed += paramString.substring(0, colonPos); - paramString = paramString.substring(colonPos, paramString.length).trim(); + default: + colonPos = paramString.indexOf(':'); + } - //if param value is wrapped in single quotes, replace with double quotes. - if (paramString[0] === '\'') { - quotePos = paramString.search(/[^\\]'/); + if (quotePos > -1) { + keys.push(paramString.substring(0, quotePos + 1).trim()); - //except for unclosed quotes to prevent infinite loops. - if (quotePos === -1) { - quotePos = paramString.length - 1; - } - else { - quotePos += 2; - } + //truncate the beginning from paramString and look for a value + paramString = paramString.substring(quotePos + 1, paramString.length).trim(); - //prepare param value for move to paramStringWellFormed var. - paramStringTmp = paramString.substring(0, quotePos); + //unset quotePos + quotePos = -1; - //unescape any escaped single quotes. - paramStringTmp = paramStringTmp.replace(/\\'/g, '\''); + } else if (colonPos > -1) { + keys.push(paramString.substring(0, colonPos).trim()); - //escape any double quotes. - paramStringTmp = paramStringTmp.replace(/"/g, '\\"'); + //truncate the beginning from paramString and look for a value + paramString = paramString.substring(colonPos, paramString.length); - //replace the delimiting single quotes with double quotes. - paramStringTmp = paramStringTmp.replace(/^'/, '"'); - paramStringTmp = paramStringTmp.replace(/'$/, '"'); + //unset colonPos + colonPos = -1; - //move param key to paramStringWellFormed var. - paramStringWellFormed += paramStringTmp; - paramString = paramString.substring(quotePos, paramString.length).trim(); + //if there are no more colons, and we're looking for a key, there is + //probably a problem. stop any further processing. + } else { + paramString = ''; + break; + } } - //if param value is wrapped in double quotes, just move to paramStringWellFormed var. - else if (paramString[0] === '"') { - quotePos = paramString.search(/[^\\]"/); - - //except for unclosed quotes to prevent infinite loops. - if (quotePos === -1) { - quotePos = paramString.length - 1; + //now, search for a value + if (paramString[0] === ':') { + paramString = paramString.substring(1, paramString.length).trim(); + + //the only reason we're using regexes here, instead of indexOf(), is + //because we don't know if the next delimiter is going to be a comma or + //a closing curly brace. since it's not much of a performance hit to + //use regexes as sparingly as here, and it's much more concise and + //readable, we'll use a regex for match() and replace() instead of + //performing conditional logic with indexOf(). + switch (paramString[0]) { + + //since a quote of same type as its wrappers would be escaped, and we + //escaped those even further with their unicodes, it is safe to look + //for wrapper pairs and conclude that their contents are values + case '"': + regex = /^"(.|\s)*?"/; + break; + case '\'': + regex = /^'(.|\s)*?'/; + break; + + //if there is no value wrapper, regex for alphanumerics, decimal + //points, and minus signs for exponential notation. + default: + regex = /^[\w\-\.]*/; } - else { - quotePos += 2; + values.push(paramString.match(regex)[0].trim()); + + //truncate the beginning from paramString and continue either + //looking for a key, or returning + paramString = paramString.replace(regex, '').trim(); + + //exit do while if the final char is '}' + if (paramString === '}') { + paramString = ''; + break; } - //move param key to paramStringWellFormed var. - paramStringWellFormed += paramString.substring(0, quotePos); - paramString = paramString.substring(quotePos, paramString.length).trim(); + //if there are no more colons, and we're looking for a value, there is + //probably a problem. stop any further processing. + } else { + paramString = ''; + break; } + } while (paramString); - //if param value is not wrapped in quotes, move everthing up to the delimiting comma to paramStringWellFormed var. - else { - delimitPos = paramString.indexOf(','); - - //except to prevent infinite loops. - if (delimitPos === -1) { - delimitPos = paramString.length - 1; + //build paramStringWellFormed string for JSON parsing + paramStringWellFormed = '{'; + for (var i = 0; i < keys.length; i++) { + + //keys + //replace single-quote wrappers with double-quotes + if (keys[i][0] === '\'' && keys[i][keys[i].length - 1] === '\'') { + paramStringWellFormed += '"'; + + //any enclosed double-quotes must be escaped + paramStringWellFormed += keys[i].substring(1, keys[i].length - 1).replace(/"/g, '\\"'); + paramStringWellFormed += '"'; + } else { + + //open wrap with double-quotes if no wrapper + if (keys[i][0] !== '"' && keys[i][0] !== '\'') { + paramStringWellFormed += '"'; + + //this is to clean up vestiges from Pattern Lab PHP's escaping scheme. + //F.Y.I. Pattern Lab PHP would allow special characters like question + //marks in parameter keys so long as the key was unwrapped and the + //special character escaped with a backslash. In Node, we need to wrap + //those keys and unescape those characters. + keys[i] = keys[i].replace(/\\/g, ''); } - else { - delimitPos += 1; + + paramStringWellFormed += keys[i]; + + //close wrap with double-quotes if no wrapper + if (keys[i][keys[i].length - 1] !== '"' && keys[i][keys[i].length - 1] !== '\'') { + paramStringWellFormed += '"'; } - paramStringWellFormed += paramString.substring(0, delimitPos); - paramString = paramString.substring(delimitPos, paramString.length).trim(); } - //break at the end. - if (paramString.length === 1) { - paramStringWellFormed += paramString.trim(); - paramString = ''; - break; + //colon delimiter. + paramStringWellFormed += ':'; + values[i]; + + //values + //replace single-quote wrappers with double-quotes + if (values[i][0] === '\'' && values[i][values[i].length - 1] === '\'') { + paramStringWellFormed += '"'; + + //any enclosed double-quotes must be escaped + paramStringWellFormed += values[i].substring(1, values[i].length - 1).replace(/"/g, '\\"'); + paramStringWellFormed += '"'; + + //for everything else, just add the value however it's wrapped + } else { + paramStringWellFormed += values[i]; } - } while (paramString); + //comma delimiter + if (i < keys.length - 1) { + paramStringWellFormed += ','; + } + } + paramStringWellFormed += '}'; + + //unescape escaped unicode except for double-quotes + paramStringWellFormed = paramStringWellFormed.replace(/\\u0027/g, '\''); + paramStringWellFormed = paramStringWellFormed.replace(/\\u0058/g, ':'); return paramStringWellFormed; } @@ -140,7 +260,7 @@ var parameter_hunter = function () { //strip out the additional data, convert string to JSON. var leftParen = pMatch.indexOf('('); - var rightParen = pMatch.indexOf(')'); + var rightParen = pMatch.lastIndexOf(')'); var paramString = '{' + pMatch.substring(leftParen + 1, rightParen) + '}'; var paramStringWellFormed = paramToJson(paramString); @@ -149,11 +269,12 @@ var parameter_hunter = function () { var localData = {}; try { - paramData = JSON.parse(paramStringWellFormed); - globalData = JSON.parse(JSON.stringify(patternlab.data)); - localData = JSON.parse(JSON.stringify(pattern.jsonFileData || {})); - } catch (e) { - console.log(e); + paramData = JSON5.parse(paramStringWellFormed); + globalData = JSON5.parse(JSON5.stringify(patternlab.data)); + localData = JSON5.parse(JSON5.stringify(pattern.jsonFileData || {})); + } catch (err) { + console.log('There was an error parsing JSON for ' + pattern.abspath); + console.log(err); } var allData = pattern_assembler.merge_data(globalData, localData); diff --git a/core/lib/pattern_assembler.js b/core/lib/pattern_assembler.js index 989b58d3d..f07c87ab4 100644 --- a/core/lib/pattern_assembler.js +++ b/core/lib/pattern_assembler.js @@ -332,10 +332,11 @@ var pattern_assembler = function () { } function parseDataLinksHelper(patternlab, obj, key) { + var JSON5 = require('json5'); var linkRE, dataObjAsString, linkMatches, expandedLink; linkRE = /link\.[A-z0-9-_]+/g; - dataObjAsString = JSON.stringify(obj); + dataObjAsString = JSON5.stringify(obj); linkMatches = dataObjAsString.match(linkRE); if (linkMatches) { @@ -349,7 +350,16 @@ var pattern_assembler = function () { } } } - return JSON.parse(dataObjAsString); + + var dataObj; + try { + dataObj = JSON5.parse(dataObjAsString); + } catch (err) { + console.log('There was an error parsing JSON for ' + key); + console.log(err); + } + + return dataObj; } //look for pattern links included in data files. diff --git a/core/lib/patternlab.js b/core/lib/patternlab.js index 8dc40732f..affb9767b 100644 --- a/core/lib/patternlab.js +++ b/core/lib/patternlab.js @@ -12,6 +12,7 @@ var patternlab_engine = function (config) { 'use strict'; var path = require('path'), + JSON5 = require('json5'), fs = require('fs-extra'), diveSync = require('diveSync'), of = require('./object_factory'), @@ -180,18 +181,24 @@ var patternlab_engine = function (config) { //json stringify lineage and lineageR var lineageArray = []; for (var i = 0; i < pattern.lineage.length; i++) { - lineageArray.push(JSON.stringify(pattern.lineage[i])); + lineageArray.push(JSON5.stringify(pattern.lineage[i])); } pattern.lineage = lineageArray; var lineageRArray = []; for (var i = 0; i < pattern.lineageR.length; i++) { - lineageRArray.push(JSON.stringify(pattern.lineageR[i])); + lineageRArray.push(JSON5.stringify(pattern.lineageR[i])); } pattern.lineageR = lineageRArray; //render the pattern, but first consolidate any data we may have - var allData = JSON.parse(JSON.stringify(patternlab.data)); + var allData; + try { + allData = JSON5.parse(JSON5.stringify(patternlab.data)); + } catch (err) { + console.log('There was an error parsing JSON for ' + pattern.abspath); + console.log(err); + } allData = pattern_assembler.merge_data(allData, pattern.jsonFileData); //also add the cachebuster value. slight chance this could collide with a user that has defined cacheBuster as a value @@ -563,11 +570,11 @@ var patternlab_engine = function (config) { //patternPaths var patternPathsTemplate = fs.readFileSync(path.resolve(paths.source.patternlabFiles, 'templates/partials/patternPaths.mustache'), 'utf8'); - var patternPathsPartialHtml = pattern_assembler.renderPattern(patternPathsTemplate, {'patternPaths': JSON.stringify(patternlab.patternPaths)}); + var patternPathsPartialHtml = pattern_assembler.renderPattern(patternPathsTemplate, {'patternPaths': JSON5.stringify(patternlab.patternPaths)}); //viewAllPaths var viewAllPathsTemplate = fs.readFileSync(path.resolve(paths.source.patternlabFiles, 'templates/partials/viewAllPaths.mustache'), 'utf8'); - var viewAllPathsPartialHtml = pattern_assembler.renderPattern(viewAllPathsTemplate, {'viewallpaths': JSON.stringify(patternlab.viewAllPaths)}); + var viewAllPathsPartialHtml = pattern_assembler.renderPattern(viewAllPathsTemplate, {'viewallpaths': JSON5.stringify(patternlab.viewAllPaths)}); //render the patternlab template, with all partials var patternlabSiteHtml = pattern_assembler.renderPattern(patternlabSiteTemplate, { diff --git a/package.gulp.json b/package.gulp.json index 18aef38d1..25e02c26c 100644 --- a/package.gulp.json +++ b/package.gulp.json @@ -9,6 +9,7 @@ "fs-extra": "^0.26.5", "glob": "^7.0.0", "html-entities": "^1.2.0", + "json5": "^0.5.0", "mustache": "^2.2.1" }, "devDependencies": { diff --git a/package.json b/package.json index 2254aef16..7a87732e5 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "fs-extra": "^0.26.2", "glob": "^7.0.0", "html-entities": "^1.2.0", + "json5": "^0.5.0", "matchdep": "^1.0.0", "mustache": "^2.2.0" }, diff --git a/test/parameter_hunter_tests.js b/test/parameter_hunter_tests.js index 96d345792..3a9abb25a 100644 --- a/test/parameter_hunter_tests.js +++ b/test/parameter_hunter_tests.js @@ -230,8 +230,100 @@ parameter_hunter.find_parameters(currentPattern, patternlab); test.equals(currentPattern.extendedTemplate, '

true not}"true"

'); + test.done(); + }, + + 'parameter hunter parses parameters with combination of quoting schemes for keys and values' : function(test){ + var currentPattern = currentPatternClosure(); + var patternlab = patternlabClosure(); + var parameter_hunter = new ph(); + + currentPattern.template = "{{> molecules-single-comment(description: true, 'foo': false, \"bar\": false, 'single': true, 'singlesingle': 'true', 'singledouble': \"true\", \"double\": true, \"doublesingle\": 'true', \"doubledouble\": \"true\") }}"; + currentPattern.extendedTemplate = currentPattern.template; + currentPattern.parameteredPartials[0] = currentPattern.template; + + parameter_hunter.find_parameters(currentPattern, patternlab); + test.equals(currentPattern.extendedTemplate, '

true

'); + + test.done(); + }, + + 'parameter hunter parses parameters with values containing a closing parenthesis' : function(test){ + // From issue #291 https://github.com/pattern-lab/patternlab-node/issues/291 + var currentPattern = currentPatternClosure(); + var patternlab = patternlabClosure(); + var parameter_hunter = new ph(); + + currentPattern.template = "{{> molecules-single-comment(description: 'Hello ) World') }}"; + currentPattern.extendedTemplate = currentPattern.template; + currentPattern.parameteredPartials[0] = currentPattern.template; + + parameter_hunter.find_parameters(currentPattern, patternlab); + test.equals(currentPattern.extendedTemplate, '

Hello ) World

'); + + test.done(); + }, + + 'parameter hunter parses parameters that follow a non-quoted value' : function(test){ + // From issue #291 https://github.com/pattern-lab/patternlab-node/issues/291 + var currentPattern = currentPatternClosure(); + var patternlab = patternlabClosure(); + var parameter_hunter = new ph(); + + patternlab.patterns[0].template = "

{{foo}}

{{bar}}

"; + patternlab.patterns[0].extendedTemplate = patternlab.patterns[0].template; + + currentPattern.template = "{{> molecules-single-comment(foo: true, bar: \"Hello World\") }}"; + currentPattern.extendedTemplate = currentPattern.template; + currentPattern.parameteredPartials[0] = currentPattern.template; + + parameter_hunter.find_parameters(currentPattern, patternlab); + test.equals(currentPattern.extendedTemplate, '

true

Hello World

'); + + test.done(); + }, + + 'parameter hunter parses parameters whose keys contain escaped quotes' : function(test){ + // From issue #291 https://github.com/pattern-lab/patternlab-node/issues/291 + var currentPattern = currentPatternClosure(); + var patternlab = patternlabClosure(); + var parameter_hunter = new ph(); + + patternlab.patterns[0].template = "

{{ silly'key }}

{{bar}}

{{ another\"silly-key }}

"; + patternlab.patterns[0].extendedTemplate = patternlab.patterns[0].template; + + currentPattern.template = "{{> molecules-single-comment('silly\\\'key': true, bar: \"Hello World\", \"another\\\"silly-key\": 42 ) }}"; + currentPattern.extendedTemplate = currentPattern.template; + currentPattern.parameteredPartials[0] = currentPattern.template; + + parameter_hunter.find_parameters(currentPattern, patternlab); + test.equals(currentPattern.extendedTemplate, '

true

Hello World

42

'); + + test.done(); + }, + + 'parameter hunter skips malformed parameters' : function(test){ + // From issue #291 https://github.com/pattern-lab/patternlab-node/issues/291 + var currentPattern = currentPatternClosure(); + var patternlab = patternlabClosure(); + var parameter_hunter = new ph(); + + patternlab.patterns[0].template = "

{{foo}}

"; + patternlab.patterns[0].extendedTemplate = patternlab.patterns[0].template; + + currentPattern.abspath = __filename; + currentPattern.template = "{{> molecules-single-comment( missing-val: , : missing-key, : , , foo: \"Hello World\") }}"; + currentPattern.extendedTemplate = currentPattern.template; + currentPattern.parameteredPartials[0] = currentPattern.template; + + console.log('\nPattern Lab should catch JSON.parse() errors and output useful debugging information...'); + parameter_hunter.find_parameters(currentPattern, patternlab); + test.equals(currentPattern.extendedTemplate, '

'); + test.done(); } + + }; }());