Allow multiple files on the command line, all of which get combined into one output. Not ready. Still needs change to source-map generation, for topLevelInitializer to be an array, and to make moving imports to the top of all topLevelInitializers (keeping track of source info) a JS-specific pass.

Author: hildjj

bin/peggy-cli.js

@@ -94,8 +94,8 @@ class PeggyCLI extends Command {
 
     /** @type {peggy.BuildOptionsBase} */
     this.argv = {};
-    /** @type {string?} */
-    this.inputFile = null;
+    /** @type {string[]} */
+    this.inputFiles = [];
     /** @type {string?} */
     this.outputFile = null;
     /** @type {object} */
@@ -115,7 +115,7 @@ class PeggyCLI extends Command {
 
     this
       .version(peggy.VERSION, "-v, --version")
-      .argument("[input_file]", 'Grammar file to read.  Use "-" to read stdin.', "-")
+      .argument("[input_file...]", 'Grammar file(s) to read.  Use "-" to read stdin.  If multiple files are given, they are combined in the given order to produce a single output.', ["-"])
       .allowExcessArguments(false)
       .addOption(
         new Option(
@@ -214,8 +214,8 @@ class PeggyCLI extends Command {
           .hideHelp()
           .default(false)
       )
-      .action((inputFile, opts) => { // On parse()
-        this.inputFile = inputFile;
+      .action((inputFiles, opts) => { // On parse()
+        this.inputFiles = inputFiles;
         this.argv = opts;
 
         if ((typeof this.argv.startRule === "string")
@@ -284,21 +284,24 @@ class PeggyCLI extends Command {
         this.argv.output = "source";
         if ((this.args.length === 0) && this.progOptions.input) {
           // Allow command line to override config file.
-          this.inputFile = this.progOptions.input;
+          this.inputFiles = Array.isArray(this.progOptions.input)
+            ? this.progOptions.input
+            : [this.progOptions.input];
         }
         this.outputFile = this.progOptions.output;
         this.outputJS = this.progOptions.output;
 
-        if ((this.inputFile === "-") && this.argv.watch) {
+        if ((this.inputFiles.indexOf("-") !== -1) && this.argv.watch) {
           this.argv.watch = false; // Make error throw.
           this.error("Can't watch stdin");
         }
 
         if (!this.outputFile) {
-          if (this.inputFile !== "-") {
-            this.outputJS = this.inputFile.slice(
+          if (this.inputFiles.indexOf("-") === -1) {
+            this.outputJS = this.inputFiles[0].slice(
               0,
-              this.inputFile.length - path.extname(this.inputFile).length
+              this.inputFiles[0].length
+                - path.extname(this.inputFiles[0]).length
             ) + ".js";
 
             this.outputFile = ((typeof this.progOptions.test !== "string")
@@ -345,7 +348,7 @@ class PeggyCLI extends Command {
         }
         this.verbose("PARSER OPTIONS:", this.argv);
         this.verbose("PROGRAM OPTIONS:", this.progOptions);
-        this.verbose('INPUT: "%s"', this.inputFile);
+        this.verbose('INPUT: "%s"', this.inputFiles);
         this.verbose('OUTPUT: "%s"', this.outputFile);
         if (this.progOptions.verbose) {
           this.argv.info = (pass, msg) => PeggyCLI.print(this.std.err, `INFO(${pass}): ${msg}`);
@@ -561,7 +564,7 @@ class PeggyCLI extends Command {
       if (this.testFile === "-") {
         this.testText = await readStream(this.std.in);
       } else {
-        this.testText = fs.readFileSync(this.testFile, "utf8");
+        this.testText = await fs.promises.readFile(this.testFile, "utf8");
       }
     }
     if (typeof this.testText === "string") {
@@ -629,26 +632,29 @@ class PeggyCLI extends Command {
    * @returns {Promise<number>}
    */
   async run() {
-    let inputStream = undefined;
-
-    if (this.inputFile === "-") {
-      this.std.in.resume();
-      inputStream = this.std.in;
-      this.argv.grammarSource = "stdin";
-    } else {
-      this.argv.grammarSource = this.inputFile;
-      inputStream = fs.createReadStream(this.inputFile);
-    }
+    const sources = [];
 
     let exitCode = 1;
     let errorText = "";
-    let input = "";
     try {
-      this.verbose("CLI", errorText = "reading input stream");
-      input = await readStream(inputStream);
+      for (const source of this.inputFiles) {
+        const input = { source, text: null };
+        this.verbose("CLI", errorText = `reading input "${source}"`);
+        if (source === "-") {
+          input.source = "stdin";
+          this.std.in.resume();
+          input.text = await readStream(this.std.in);
+        } else {
+          input.text = await fs.promises.readFile(source, "utf8");
+        }
+        sources.push(input);
+      }
+
+      // This is wrong.  It's a hack in place until source generation is fixed.
+      this.argv.grammarSource = sources[0].source;
 
       this.verbose("CLI", errorText = "parsing grammar");
-      const source = peggy.generate(input, this.argv); // All of the real work.
+      const source = peggy.generate(sources, this.argv); // All of the real work.
 
       this.verbose("CLI", errorText = "open output stream");
       const outputStream = await this.openOutputStream();
@@ -669,10 +675,6 @@ class PeggyCLI extends Command {
         await this.test(mappedSource);
       }
     } catch (error) {
-      const sources = [{
-        source: this.argv.grammarSource,
-        text: input,
-      }];
       if (this.testGrammarSource) {
         sources.push({
           source: this.testGrammarSource,
@@ -711,19 +713,21 @@ class PeggyCLI extends Command {
     if (this.argv.watch) {
       const Watcher = require("./watcher.js"); // Lazy: usually not needed.
       const hasTest = this.progOptions.test || this.progOptions.testFile;
+      const watchFiles = [...this.inputFiles];
+      if (this.progOptions.testFile) {
+        watchFiles.push(this.progOptions.testFile);
+      }
+      this.watcher = new Watcher(...watchFiles);
 
-      this.watcher = new Watcher(this.inputFile);
-
-      const that = this;
-      this.watcher.on("change", async() => {
-        PeggyCLI.print(this.std.err, `"${that.inputFile}" changed...`);
+      this.watcher.on("change", async fn => {
+        PeggyCLI.print(this.std.err, `"${fn}" changed...`);
         this.lastError = null;
-        await that.run();
+        await this.run();
 
-        if (that.lastError) {
-          PeggyCLI.print(this.std.err, that.lastError);
+        if (this.lastError) {
+          PeggyCLI.print(this.std.err, this.lastError);
         } else if (!hasTest) {
-          PeggyCLI.print(this.std.err, `Wrote: "${that.outputFile}"`);
+          PeggyCLI.print(this.std.err, `Wrote: "${this.outputFile}"`);
         }
       });
 

bin/watcher.js

@@ -6,6 +6,8 @@ const { EventEmitter } = require("events");
 
 // This may have to be tweaked based on experience.
 const DEBOUNCE_MS = 100;
+const CLOSING = Symbol("CLOSING");
+const ERROR = Symbol("ERROR");
 
 /**
  * Relatively feature-free file watcher that deals with some of the
@@ -18,51 +20,70 @@ const DEBOUNCE_MS = 100;
  */
 class Watcher extends EventEmitter {
   /**
-   * Creates an instance of Watcher.
+   * Creates an instance of Watcher.  This only works for files in a small
+   * number of directories.
    *
-   * @param {string} filename The file to watch.  Should be a plain file,
-   *   not a directory, pipe, etc.
+   * @param {string[]} filenames The files to watch.  Should be one or more
+   *   strings, each of which is the name of a plain file, not a directory,
+   *   pipe, etc.
    */
-  constructor(filename) {
+  constructor(...filenames) {
     super();
 
-    const rfile = path.resolve(filename);
-    const { dir, base } = path.parse(rfile);
-    let timeout = null;
+    const resolved = new Set(filenames.map(fn => path.resolve(fn)));
+    const dirs = new Set([...resolved].map(fn => path.dirname(fn)));
 
-    // eslint-disable-next-line func-style -- Needs this.
-    const changed = (typ, fn) => {
-      if (fn === base) {
-        if (!timeout) {
-          fs.stat(rfile, (er, stats) => {
-            if (!er && stats.isFile()) {
-              this.emit("change", stats);
-            }
-          });
-        } else {
-          clearTimeout(timeout);
+    this.timeout = null;
+    this.watchers = [];
+
+    for (const dir of dirs) {
+      // eslint-disable-next-line func-style -- Needs "this"
+      const changed = (_typ, fn) => {
+        if (typeof this.timeout === "symbol") {
+          return;
         }
+        const filename = path.join(dir, fn);
+        // Might be a different fil changing in one of the target dirs
+        if (resolved.has(filename)) {
+          if (!this.timeout) {
+            fs.stat(filename, (er, stats) => {
+              if (!er && stats.isFile()) {
+                this.emit("change", filename, stats);
+              }
+            });
+          } else {
+            clearTimeout(this.timeout);
+          }
 
-        // De-bounce
-        timeout = setTimeout(() => {
-          timeout = null;
-        }, Watcher.interval);
-      }
-    };
-    const closed = () => this.emit("close");
+          // De-bounce, across all files
+          this.timeout = setTimeout(() => {
+            this.timeout = null;
+          }, Watcher.interval);
+        }
+      };
 
-    this.watcher = fs.watch(dir);
-    this.watcher.on("error", er => {
-      this.watcher.off("close", closed);
-      this.watcher.once("close", () => this.emit("error", er));
-      this.watcher.close();
-      this.watcher = null;
-    });
-    this.watcher.on("close", closed);
-    this.watcher.on("change", changed);
+      const w = fs.watch(dir);
+      w.on("error", er => {
+        const t = this.timeout;
+        this.timeout = ERROR;
+        if (t && (typeof t !== "symbol")) {
+          clearTimeout(t);
+        }
+        this.emit("error", er);
+        this.close();
+      });
+      w.on("change", changed);
+      this.watchers.push(w);
+    }
 
     // Fire initial time if file exists.
-    setImmediate(() => changed("rename", base));
+    setImmediate(() => {
+      if (this.watchers.length > 0) {
+        // First watcher will correspond to the directory of the first filename.
+        const w = this.watchers[0];
+        w.emit("change", "initial", path.basename([...resolved][0]));
+      }
+    });
   }
 
   /**
@@ -71,16 +92,31 @@ class Watcher extends EventEmitter {
    * @returns {Promise<void>} Always resolves.
    */
   close() {
-    return new Promise(resolve => {
-      if (this.watcher) {
-        this.watcher.once("close", resolve);
-        this.watcher.close();
-      } else {
-        resolve();
+    // Stop any more events from firing, immediately
+    const t = this.timeout;
+
+    if (t) {
+      if (typeof t !== "symbol") {
+        this.timeout = CLOSING;
+        clearTimeout(t);
+      }
+    }
+
+    const p = [];
+    for (const w of this.watchers) {
+      p.push(new Promise(resolve => {
+        w.once("close", resolve);
+      }));
+      w.close();
+    }
+    return Promise.all(p).then(() => {
+      this.watchers = [];
+      if (t !== ERROR) {
+        this.emit("close");
       }
-      this.watcher = null;
     });
   }
 }
+
 Watcher.interval = DEBOUNCE_MS;
 module.exports = Watcher;

lib/compiler/asts.js

@@ -86,6 +86,50 @@ const asts = {
 
     return consumes(node);
   },
+
+  combine(asts) {
+    let combined = null;
+    for (const ast of asts) {
+      if (combined) {
+        // Note: Change topLevelInitializer and initializer to be an array in
+        // order to keep the location along with.
+        if (ast.topLevelInitializer) {
+          if (combined.topLevelInitializer) {
+            // Try this and see if it's better than the other way in practice.
+            const init
+              = ast.topLevelInitializer
+                + "\n"
+                + combined.topLevelInitializer;
+
+            // Move imports to the top
+            let imports = "";
+            let nonImports = "";
+            for (const line of init.split(/[\r\n]+/)) {
+              if (/^\s*import/.test(line)) {
+                imports += line;
+              } else {
+                nonImports += line;
+              }
+            }
+            combined.topLevelInitializer = imports + "\n" + nonImports;
+          } else {
+            combined.topLevelInitializer = ast.topLevelInitializer;
+          }
+        }
+        if (ast.initializer) {
+          if (combined.initializer) {
+            combined.initializer = ast.initializer + "\n" + combined.initializer;
+          } else {
+            combined.initializer = ast.initializer;
+          }
+        }
+        combined.rules = combined.rules.concat(ast.rules);
+      } else {
+        combined = ast;
+      }
+    }
+    return combined;
+  },
 };
 
 module.exports = asts;