Introduce behavior and structure testing

fluent-syntax/makefile

@@ -22,3 +22,13 @@ compat.js: $(PACKAGE).js
 clean:
 	@rm -f $(PACKAGE).js compat.js
 	@echo -e " $(OK) clean"
+
+STRUCTURE_FTL := $(wildcard test/fixtures_structure/*.ftl)
+STRUCTURE_AST := $(STRUCTURE_FTL:.ftl=.json)
+
+fixtures: $(STRUCTURE_AST)
+
+.PHONY: $(STRUCTURE_AST)
+$(STRUCTURE_AST): test/fixtures_structure/%.json: test/fixtures_structure/%.ftl
+	@../tools/parse.js -s $< > $@
+	@echo -e " $(OK) $@"

fluent-syntax/src/ast.js

@@ -227,11 +227,15 @@ export class Span extends Node {
 }
 
 export class Annotation extends Node {
-  constructor(name, message, pos) {
+  constructor(code, args = [], message) {
     super();
     this.type = 'Annotation';
-    this.name = name;
+    this.code = code;
+    this.args = args;
     this.message = message;
-    this.pos = pos;
+  }
+
+  addSpan(start, end) {
+    this.span = new Span(start, end);
   }
 }

fluent-syntax/src/errors.js

@@ -1 +1,51 @@
-export class ParseError extends Error {}
+export class ParseError extends Error {
+  constructor(code, ...args) {
+    super();
+    this.code = code;
+    this.args = args;
+    this.message = getErrorMessage(code, args);
+  }
+}
+
+function getErrorMessage(code, args) {
+  switch (code) {
+    case 'E0001':
+      return 'Generic error';
+    case 'E0002':
+      return 'Expected an entry start';
+    case 'E0003': {
+      const [token] = args;
+      return `Expected token: "${token}"`;
+    }
+    case 'E0004': {
+      const [range] = args;
+      return `Expected a character from range: "${range}"`;
+    }
+    case 'E0005': {
+      const [id] = args;
+      return `Expected entry "${id}" to have a value, attributes or tags`;
+    }
+    case 'E0006': {
+      const [field] = args;
+      return `Expected field: "${field}"`;
+    }
+    case 'E0007':
+      return 'Keyword cannot end with a whitespace';
+    case 'E0008':
+      return 'Callee has to be a simple identifier';
+    case 'E0009':
+      return 'Key has to be a simple identifier';
+    case 'E0010':
+      return 'Expected one of the variants to be marked as default (*)';
+    case 'E0011':
+      return 'Expected at least one variant after "->"';
+    case 'E0012':
+      return 'Tags cannot be added to messages with attributes';
+    case 'E0013':
+      return 'Expected variant key';
+    case 'E0014':
+      return 'Expected literal';
+    default:
+      return code;
+  }
+}

fluent-syntax/src/ftlstream.js

@@ -43,7 +43,7 @@ export class FTLParserStream extends ParserStream {
       return true;
     }
 
-    throw new ParseError(`Expected token "${ch}"`);
+    throw new ParseError('E0003', ch);
   }
 
   takeCharIf(ch) {
@@ -167,7 +167,7 @@ export class FTLParserStream extends ParserStream {
       this.next();
       return ret;
     }
-    throw new ParseError('Expected char range');
+    throw new ParseError('E0004', 'a-zA-Z');
   }
 
   takeIDChar() {

fluent-syntax/src/parser.js

@@ -45,9 +45,8 @@ function getEntryOrJunk(ps) {
       throw err;
     }
 
-    const annot = new AST.Annotation(
-      'ParseError', err.message, ps.getIndex()
-    );
+    const annot = new AST.Annotation(err.code, err.args, err.message);
+    annot.addSpan(ps.getIndex(), ps.getIndex());
 
     ps.skipToNextEntryStart();
     const nextEntryStart = ps.getIndex();
@@ -79,7 +78,7 @@ function getEntry(ps) {
   if (comment) {
     return comment;
   }
-  throw new ParseError('Expected entry');
+  throw new ParseError('E0002');
 }
 
 function getComment(ps) {
@@ -151,13 +150,13 @@ function getMessage(ps, comment) {
 
   if (ps.isPeekNextLineTagStart()) {
     if (attrs !== undefined) {
-      throw new ParseError('Tags cannot be added to messages with attributes');
+      throw new ParseError('E0012');
     }
     tags = getTags(ps);
   }
 
   if (pattern === undefined && attrs === undefined && tags === undefined) {
-    throw new ParseError('Missing field');
+    throw new ParseError('E0005', id);
   }
 
   return new AST.Message(id, pattern, attrs, tags, comment);
@@ -183,7 +182,7 @@ function getAttributes(ps) {
     const value = getPattern(ps);
 
     if (value === undefined) {
-      throw new ParseError('Expected field');
+      throw new ParseError('E0006', 'value');
     }
 
     attrs.push(new AST.Attribute(key, value));
@@ -232,7 +231,7 @@ function getVariantKey(ps) {
   const ch = ps.current();
 
   if (!ch) {
-    throw new ParseError('Expected VariantKey');
+    throw new ParseError('E0013');
   }
 
   const cc = ch.charCodeAt(0);
@@ -271,7 +270,7 @@ function getVariants(ps) {
     const value = getPattern(ps);
 
     if (!value) {
-      throw new ParseError('Expected field');
+      throw new ParseError('E0006', 'value');
     }
 
     variants.push(new AST.Variant(key, value, defaultIndex));
@@ -282,7 +281,7 @@ function getVariants(ps) {
   }
 
   if (!hasDefault) {
-    throw new ParseError('Missing default variant');
+    throw new ParseError('E0010');
   }
 
   return variants;
@@ -314,7 +313,7 @@ function getDigits(ps) {
   }
 
   if (num.length === 0) {
-    throw new ParseError('Expected char range');
+    throw new ParseError('E0004', '0-9');
   }
 
   return num;
@@ -432,7 +431,7 @@ function getExpression(ps) {
       const variants = getVariants(ps);
 
       if (variants.length === 0) {
-        throw new ParseError('Missing variants');
+        throw new ParseError('E0011');
       }
 
       ps.expectChar('\n');
@@ -501,7 +500,7 @@ function getCallArgs(ps) {
 
     if (ps.current() === ':') {
       if (exp.type !== 'MessageReference') {
-        throw new ParseError('Forbidden key');
+        throw new ParseError('E0009');
       }
 
       ps.next();
@@ -533,7 +532,7 @@ function getArgVal(ps) {
   } else if (ps.currentIs('"')) {
     return getString(ps);
   }
-  throw new ParseError('Expected field');
+  throw new ParseError('E0006', 'value');
 }
 
 function getString(ps) {
@@ -556,7 +555,7 @@ function getLiteral(ps) {
   const ch = ps.current();
 
   if (!ch) {
-    throw new ParseError('Expected literal');
+    throw new ParseError('E0014');
   }
 
   if (ps.isNumberStart()) {

fluent-syntax/test/behavior_test.js

@@ -0,0 +1,87 @@
+import assert from 'assert';
+import { join } from 'path';
+import { readdir } from 'fs';
+import { readfile } from './util';
+import { parse } from '../src/parser';
+
+const sigil = '^\/\/~ ';
+const reDirective = new RegExp(`${sigil}(.*)[\n$]`, 'gm');
+
+function* directives(source) {
+  let match;
+  while ((match = reDirective.exec(source)) !== null) {
+    yield match[1];
+  }
+}
+
+function preprocess(source) {
+  return {
+    directives: [...directives(source)],
+    source: source.replace(reDirective, ''),
+  };
+}
+
+function getCodeName(code) {
+  switch (code[0]) {
+    case 'E':
+      return `ERROR ${code}`;
+    case 'W':
+      return `WARNING ${code}`;
+    case 'H':
+      return `HINT ${code}`;
+    default:
+      throw new Error('Unknown Annotation code');
+  }
+}
+
+function serialize(annot) {
+  const { code, args, span: { start, end } } = annot;
+  const parts = [getCodeName(code)];
+
+  if (start === end) {
+    parts.push(`pos ${start}`);
+  } else {
+    parts.push(`start ${start}`, `end ${end}`);
+  }
+
+  if (args.length) {
+    const prettyArgs = args.map(arg => `"${arg}"`).join(' ');
+    parts.push(`args ${prettyArgs}`);
+  }
+
+  return parts.join(', ');
+}
+
+function toDirectives(annots, cur) {
+  return annots.concat(cur.annotations.map(serialize));
+}
+
+const fixtures = join(__dirname, 'fixtures_behavior');
+
+readdir(fixtures, function(err, filenames) {
+  if (err) {
+    throw err;
+  }
+
+  const ftlnames = filenames.filter(
+    filename => filename.endsWith('.ftl')
+  );
+
+  suite('Behavior tests', function() {
+    for (const filename of ftlnames) {
+      const filepath = join(fixtures, filename);
+      test(filename, function() {
+        return readfile(filepath).then(file => {
+          const { directives, source } = preprocess(file);
+          const expected = directives.join('\n') + '\n';
+          const ast = parse(source);
+          const actual = ast.body.reduce(toDirectives, []).join('\n') + '\n';
+          assert.deepEqual(
+            actual, expected,
+            'Actual Annotations don\'t match the expected ones'
+          );
+        });
+      });
+    }
+  });
+});

fluent-syntax/test/fixtures_behavior/bar.ftl

@@ -0,0 +1,2 @@
+bar = Bar {
+//~ ERROR E0004, pos 11, args "a-zA-Z"

fluent-syntax/test/fixtures_behavior/foo.ftl

@@ -0,0 +1 @@
+foo = Foo

fluent-syntax/test/fixtures_structure/foo.ftl

@@ -0,0 +1 @@
+foo = Foo

fluent-syntax/test/fixtures_structure/foo.json

@@ -0,0 +1,31 @@
+{
+  "type": "Resource",
+  "body": [
+    {
+      "type": "Message",
+      "span": {
+        "type": "Span",
+        "start": 0,
+        "end": 9
+      },
+      "annotations": [],
+      "id": {
+        "type": "Identifier",
+        "name": "foo"
+      },
+      "value": {
+        "type": "Pattern",
+        "elements": [
+          {
+            "type": "TextElement",
+            "value": "Foo"
+          }
+        ]
+      },
+      "attributes": null,
+      "tags": null,
+      "comment": null
+    }
+  ],
+  "comment": null
+}

fluent-syntax/test/structure_test.js

@@ -0,0 +1,36 @@
+import assert from 'assert';
+import { join } from 'path';
+import { readdir } from 'fs';
+import { readfile } from './util';
+
+import { parse } from '../src/parser';
+
+const fixtures = join(__dirname, 'fixtures_structure');
+
+readdir(fixtures, function(err, filenames) {
+  if (err) {
+    throw err;
+  }
+
+  const ftlnames = filenames.filter(
+    filename => filename.endsWith('.ftl')
+  );
+
+  suite('Structure tests', function() {
+    for (const filename of ftlnames) {
+      const ftlpath = join(fixtures, filename);
+      const astpath = ftlpath.replace(/ftl$/, 'json');
+      test(filename, function() {
+        return Promise.all(
+          [ftlpath, astpath].map(readfile)
+        ).then(([ftl, expected]) => {
+          const ast = parse(ftl);
+          assert.deepEqual(
+            ast, JSON.parse(expected),
+            'Actual Annotations don\'t match the expected ones'
+          );
+        });
+      });
+    }
+  });
+});