v8: add v8.startupSnapshot utils

This adds several APIs under the `v8.startupSnapshot` namespace
for specifying hooks into the startup snapshot serialization
and deserialization.

- isBuildingSnapshot()
- addSerializeCallback()
- addDeserializeCallback()
- setDeserializeMainFunction()

Author: joyeecheung

doc/api/errors.md

@@ -2314,6 +2314,11 @@ has occurred when attempting to start the loop.
 Once no more items are left in the queue, the idle loop must be suspended. This
 error indicates that the idle loop has failed to stop.
 
+### `ERR_NOT_BUILDING_SNAPSHOT`
+
+An attempt was made to use operations that can only be used when building
+V8 startup snapshot even though Node.js isn't building one.
+
 <a id="ERR_NO_CRYPTO"></a>
 
 ### `ERR_NO_CRYPTO`

doc/api/v8.md

@@ -876,6 +876,131 @@ Called immediately after a promise continuation executes. This may be after a
 Called when the promise receives a resolution or rejection value. This may
 occur synchronously in the case of `Promise.resolve()` or `Promise.reject()`.
 
+## Startup Snapshot API
+
+<!-- YAML
+added: REPLACEME
+-->
+
+The `v8.startupSnapshot` interface can be used to add serialization and
+deserialization hooks for custom startup snapshots. Currently the startup
+snapshots can only be built into the Node.js binary from source.
+
+```console
+cd /path/to/node
+./configure --node-snapshot-main=entry.js
+make node
+ # This binary contains the result of the execution of entry.js
+out/Release/node
+```
+
+In the example above, `entry.js` can use methods from the `v8.startupSnapshot`
+interface to specify how to save information for custom objects in the snapshot
+during serialization and how the information can be used to synchronize these
+objects during deserialization of the snapshot. For example, if the `entry.js`
+contains the following script:
+
+```cjs
+'use strict';
+
+const fs = require('fs');
+const zlib = require('zlib');
+const path = require('path');
+const {
+  isBuildingSnapshot,
+  addSerializeCallback,
+  addDeserializeCallback,
+  setDeserializeMainFunction
+} = require('v8').startupSnapshot;
+
+const filePath = path.resolve(__dirname, '../x1024.txt');
+const storage = {};
+
+if (isBuildingSnapshot()) {
+  addSerializeCallback(({ filePath }) => {
+    storage[filePath] = zlib.gzipSync(fs.readFileSync(filePath));
+  }, { filePath });
+
+  addDeserializeCallback(({ filePath }) => {
+    storage[filePath] = zlib.gunzipSync(storage[filePath]);
+  }, { filePath });
+
+  setDeserializeMainFunction(({ filePath }) => {
+    console.log(storage[filePath].toString());
+  }, { filePath });
+}
+```
+
+The resulted binary will simply print the data deserialized from the snapshot
+during start up:
+
+```console
+out/Release/node
+# Prints content of ./test/fixtures/x1024.txt
+```
+
+### `v8.startupSnapshot.addSerializeCallback(callback[, data])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `callback` {Function} Callback to be invoked before serialization.
+* `data` {any} Optional data that will be pass to the `callback` when it
+  gets called.
+
+Add a callback that will be called when the Node.js instance is about to
+get serialized into a snapshot and exit. This can be used to release
+resources that should not or cannot be serialized or to convert user data
+into a form more suitable for serialization.
+
+### `v8.startupSnapshot.addDeserializeCallback(callback[, data])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `callback` {Function} Callback to be invoked after the snapshot is
+  deserialized.
+* `data` {any} Optional data that will be pass to the `callback` when it
+  gets called.
+
+Add a callback that will be called when the Node.js instance is deserialized
+from a snapshot. The `callback` and the `data` (if provided) will be
+serialized into the snapshot, they can be used to re-initialize the state
+of the application or to re-acquire resources that the application needs
+when the application is restarted from the snapshot.
+
+### `v8.startupSnapshot.setDeserializeMainFunction(callback[, data])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `callback` {Function} Callback to be invoked as the entry point after the
+  snapshot is deserialized.
+* `data` {any} Optional data that will be pass to the `callback` when it
+  gets called.
+
+This sets the entry point of the Node.js application when it is deserialized
+from a snapshot. This can be called only once in the snapshot building
+script. If called, the deserialized application no longer needs an additional
+entry point script to start up and will simply invoke the callback along with
+the deserialized data (if provided), otherwise an entry point script still
+needs to be provided to the deserialized application.
+
+### `v8.startupSnapshot.isBuildingSnapshot()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Returns: {boolean}
+
+Returns true if the Node.js instance is run to build a snapshot.
+
+### 
+
 [HTML structured clone algorithm]: https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm
 [Hook Callbacks]: #hook-callbacks
 [V8]: https://developers.google.com/v8/

lib/internal/bootstrap/pre_execution.js

@@ -54,7 +54,6 @@ function prepareMainThreadExecution(expandArgv1 = false,
       setupCoverageHooks(process.env.NODE_V8_COVERAGE);
   }
 
-
   setupDebugEnv();
 
   // Print stack trace on `SIGINT` if option `--trace-sigint` presents.
@@ -85,6 +84,8 @@ function prepareMainThreadExecution(expandArgv1 = false,
   initializeDeprecations();
   initializeWASI();
 
+  require('internal/v8/startup_snapshot').runDeserializeCallbacks();
+
   if (!initialzeModules) {
     return;
   }

lib/internal/bootstrap/switches/is_main_thread.js

@@ -2,7 +2,10 @@
 
 const { ObjectDefineProperty } = primordials;
 const rawMethods = internalBinding('process_methods');
-
+const {
+  addSerializeCallback,
+  isBuildingSnapshot
+} = require('v8').startupSnapshot;
 // TODO(joyeecheung): deprecate and remove these underscore methods
 process._debugProcess = rawMethods._debugProcess;
 process._debugEnd = rawMethods._debugEnd;
@@ -133,6 +136,12 @@ function refreshStderrOnSigWinch() {
   stderr._refreshSize();
 }
 
+function addCleanup(fn) {
+  if (isBuildingSnapshot()) {
+    addSerializeCallback(fn)
+  }
+}
+
 function getStdout() {
   if (stdout) return stdout;
   stdout = createWritableStdioStream(1);
@@ -144,12 +153,14 @@ function getStdout() {
     process.on('SIGWINCH', refreshStdoutOnSigWinch);
   }
 
-  internalBinding('mksnapshot').cleanups.push(function cleanupStdout() {
+  addCleanup(function cleanupStdout() {
     stdout._destroy = stdoutDestroy;
     stdout.destroy();
     process.removeListener('SIGWINCH', refreshStdoutOnSigWinch);
     stdout = undefined;
   });
+  // No need to add deserialize callback because stdout = undefined above
+  // causes the stream to be lazily initialized again later.
   return stdout;
 }
 
@@ -163,12 +174,14 @@ function getStderr() {
   if (stderr.isTTY) {
     process.on('SIGWINCH', refreshStderrOnSigWinch);
   }
-  internalBinding('mksnapshot').cleanups.push(function cleanupStderr() {
+  addCleanup(function cleanupStderr() {
     stderr._destroy = stderrDestroy;
     stderr.destroy();
     process.removeListener('SIGWINCH', refreshStderrOnSigWinch);
     stderr = undefined;
   });
+  // No need to add deserialize callback because stderr = undefined above
+  // causes the stream to be lazily initialized again later.
   return stderr;
 }
 
@@ -255,10 +268,12 @@ function getStdin() {
     }
   }
 
-  internalBinding('mksnapshot').cleanups.push(function cleanupStdin() {
+  addCleanup(function cleanupStdin() {
     stdin.destroy();
     stdin = undefined;
   });
+  // No need to add deserialize callback because stdin = undefined above
+  // causes the stream to be lazily initialized again later.
   return stdin;
 }
 

lib/internal/errors.js

@@ -1445,6 +1445,8 @@ E('ERR_NETWORK_IMPORT_BAD_RESPONSE',
   "import '%s' received a bad response: %s", Error);
 E('ERR_NETWORK_IMPORT_DISALLOWED',
   "import of '%s' by %s is not supported: %s", Error);
+E('ERR_NOT_BUILDING_SNAPSHOT',
+  'Operation cannot be invoked when not building startup snapshot', Error);
 E('ERR_NO_CRYPTO',
   'Node.js is not compiled with OpenSSL crypto support', Error);
 E('ERR_NO_ICU',

lib/internal/main/mksnapshot.js

@@ -9,7 +9,7 @@ const {
 const binding = internalBinding('mksnapshot');
 const { NativeModule } = require('internal/bootstrap/loaders');
 const {
-  compileSnapshotMain,
+  compileSerializeMain,
 } = binding;
 
 const {
@@ -83,7 +83,7 @@ const supportedModules = new SafeSet(new SafeArrayIterator([
   'v8',
   // 'vm',
   // 'worker_threads',
-  // 'zlib',
+  'zlib',
 ]));
 
 const warnedModules = new SafeSet();
@@ -117,25 +117,22 @@ function main() {
   } = require('internal/bootstrap/pre_execution');
 
   prepareMainThreadExecution(true, false);
-  process.once('beforeExit', function runCleanups() {
-    for (const cleanup of binding.cleanups) {
-      cleanup();
-    }
-  });
 
   const file = process.argv[1];
   const path = require('path');
   const filename = path.resolve(file);
   const dirname = path.dirname(filename);
   const source = readFileSync(file, 'utf-8');
-  const snapshotMainFunction = compileSnapshotMain(filename, source);
+  const serializeMainFunction = compileSerializeMain(filename, source);
+
+  require('internal/v8/startup_snapshot').initializeCallbacks();
 
   if (getOptionValue('--inspect-brk')) {
     internalBinding('inspector').callAndPauseOnStart(
-      snapshotMainFunction, undefined,
+      serializeMainFunction, undefined,
       requireForUserSnapshot, filename, dirname);
   } else {
-    snapshotMainFunction(requireForUserSnapshot, filename, dirname);
+    serializeMainFunction(requireForUserSnapshot, filename, dirname);
   }
 }
 

lib/internal/v8/startup_snapshot.js

@@ -0,0 +1,108 @@
+"use strict";
+
+const {
+  validateFunction
+} = require('internal/validators');
+const {
+  ERR_NOT_BUILDING_SNAPSHOT
+} = require('internal/errors');
+
+const {
+  setSerializeCallback,
+  setDeserializeCallback,
+  setDeserializeMainFunction: _setDeserializeMainFunction,
+  markBootstrapComplete
+} = internalBinding('mksnapshot');
+
+function isBuildingSnapshot() {
+  // For now this is the only way to build a snapshot.
+  return require('internal/options').getOptionValue('--build-snapshot');
+}
+
+function throwIfNotBuildingSnapshot() {
+  if (!isBuildingSnapshot()) {
+    throw new ERR_NOT_BUILDING_SNAPSHOT();
+  }
+}
+
+const deserializeCallbacks = [];
+let deserializeCallbackIsSet = false;
+function runDeserializeCallbacks() {
+  while (deserializeCallbacks.length > 0) {
+    const [callback, data] = deserializeCallbacks.shift();
+    callback(data);
+  }
+}
+function addDeserializeCallback(callback, data) {
+  throwIfNotBuildingSnapshot();
+  validateFunction(callback, 'callback');
+  if (!deserializeCallbackIsSet) {
+    // TODO(joyeecheung): when the main function handling is done in JS,
+    // the deserialize callbacks can always be invoked. For now only
+    // store it in C++ when it's actually used to avoid unnecessary
+    // C++ -> JS costs.
+    setDeserializeCallback(runDeserializeCallbacks);
+    deserializeCallbackIsSet = true;
+  }
+  deserializeCallbacks.push([callback, data]);
+}
+
+const serializeCallbacks = [];
+function runSerializeCallbacks() {
+  while (serializeCallbacks.length > 0) {
+    const [callback, data] = serializeCallbacks.shift();
+    callback(data);
+  }
+  // Remove the hooks from the snapshot.
+  require('v8').startupSnapshot = undefined;
+}
+function addSerializeCallback(callback, data) {
+  throwIfNotBuildingSnapshot();
+  validateFunction(callback, 'callback');
+  serializeCallbacks.push([callback, data]);
+}
+
+function initializeCallbacks() {
+  // Only run the serialize callbacks in snapshot building mode, otherwise
+  // they throw.
+  if (isBuildingSnapshot()) {
+    setSerializeCallback(runSerializeCallbacks);
+  }
+}
+
+let deserializeMainIsSet = false;
+function setDeserializeMainFunction(callback, data) {
+  throwIfNotBuildingSnapshot();
+  // TODO(joyeecheung): In lib/internal/bootstrap/node.js, create a default
+  // main function to run the lib/internal/main scripts and make sure that
+  // the main function set in the snapshot building process takes precedence.
+  validateFunction(callback, 'callback');
+  if (deserializeMainIsSet) {
+    throw new Error('Deserialize main function is already configured.');
+  }
+  deserializeMainIsSet = true;
+
+  _setDeserializeMainFunction(function deserializeMain() {
+    const {
+      prepareMainThreadExecution
+    } = require('internal/bootstrap/pre_execution');
+
+    // This should be in sync with run_main_module.js until we make that
+    // a built-in main function.
+    prepareMainThreadExecution(true);
+    markBootstrapComplete();
+    callback(data);
+  });
+}
+
+module.exports = {
+  initializeCallbacks,
+  runDeserializeCallbacks,
+  // exposed to require('v8').startupSnapshot
+  namespace: {
+    addDeserializeCallback,
+    addSerializeCallback,
+    setDeserializeMainFunction,
+    isBuildingSnapshot
+  }
+};