bootstrap: make snapshot reproducible

This patch uses the new V8 API to {de}serialize context slots for
snapshot in order to make the snapshot reproducible. Also
added a test for the reproducibility of snapshots.

Author: joyeecheung

src/api/environment.cc

@@ -458,7 +458,13 @@ Environment* CreateEnvironment(
   if (use_snapshot) {
     context = Context::FromSnapshot(isolate,
                                     SnapshotData::kNodeMainContextIndex,
-                                    {DeserializeNodeInternalFields, env})
+                                    v8::DeserializeInternalFieldsCallback(
+                                        DeserializeNodeInternalFields, env),
+                                    nullptr,
+                                    MaybeLocal<Value>(),
+                                    nullptr,
+                                    v8::DeserializeContextDataCallback(
+                                        DeserializeNodeContextData, env))
                   .ToLocalChecked();
 
     CHECK(!context.IsEmpty());

src/node_snapshotable.cc

@@ -1155,8 +1155,11 @@ ExitCode SnapshotBuilder::CreateSnapshot(SnapshotData* out,
     CHECK_EQ(index, SnapshotData::kNodeVMContextIndex);
     index = creator->AddContext(base_context);
     CHECK_EQ(index, SnapshotData::kNodeBaseContextIndex);
-    index = creator->AddContext(main_context,
-                                {SerializeNodeContextInternalFields, env});
+    index = creator->AddContext(
+        main_context,
+        v8::SerializeInternalFieldsCallback(SerializeNodeContextInternalFields,
+                                            env),
+        v8::SerializeContextDataCallback(SerializeNodeContextData, env));
     CHECK_EQ(index, SnapshotData::kNodeMainContextIndex);
   }
 
@@ -1255,6 +1258,17 @@ std::string SnapshotableObject::GetTypeName() const {
   }
 }
 
+void DeserializeNodeContextData(Local<Context> holder,
+                                int index,
+                                StartupData payload,
+                                void* callback_data) {
+  DCHECK(index == ContextEmbedderIndex::kEnvironment ||
+         index == ContextEmbedderIndex::kRealm ||
+         index == ContextEmbedderIndex::kContextTag);
+  // This is a no-op for now. We will reset all the pointers in
+  // Environment::AssignToContext() via the realm constructor.
+}
+
 void DeserializeNodeInternalFields(Local<Object> holder,
                                    int index,
                                    StartupData payload,
@@ -1320,6 +1334,44 @@ void DeserializeNodeInternalFields(Local<Object> holder,
   }
 }
 
+StartupData SerializeNodeContextData(Local<Context> holder,
+                                     int index,
+                                     void* callback_data) {
+  DCHECK(index == ContextEmbedderIndex::kEnvironment ||
+         index == ContextEmbedderIndex::kContextifyContext ||
+         index == ContextEmbedderIndex::kRealm ||
+         index == ContextEmbedderIndex::kContextTag);
+  void* data = holder->GetAlignedPointerFromEmbedderData(index);
+  per_process::Debug(DebugCategory::MKSNAPSHOT,
+                     "Serialize context data, index=%d, holder=%p, ptr=%p\n",
+                     static_cast<int>(index),
+                     *holder,
+                     data);
+  // Serialization of contextify context is not yet supported.
+  if (index == ContextEmbedderIndex::kContextifyContext) {
+    DCHECK_NULL(data);
+    return {nullptr, 0};
+  }
+
+  // We need to use use new[] because V8 calls delete[] on the returned data.
+  int size = sizeof(ContextEmbedderIndex);
+  char* result = new char[size];
+  ContextEmbedderIndex* index_data =
+      reinterpret_cast<ContextEmbedderIndex*>(result);
+  *index_data = static_cast<ContextEmbedderIndex>(index);
+
+  // For now we just reset all of them in Environment::AssignToContext()
+  switch (index) {
+    case ContextEmbedderIndex::kEnvironment:
+    case ContextEmbedderIndex::kContextifyContext:
+    case ContextEmbedderIndex::kRealm:
+    case ContextEmbedderIndex::kContextTag:
+      return StartupData{result, size};
+    default:
+      UNREACHABLE();
+  }
+}
+
 StartupData SerializeNodeContextInternalFields(Local<Object> holder,
                                                int index,
                                                void* callback_data) {

src/node_snapshotable.h

@@ -126,10 +126,17 @@ class SnapshotableObject : public BaseObject {
 v8::StartupData SerializeNodeContextInternalFields(v8::Local<v8::Object> holder,
                                                    int index,
                                                    void* env);
+v8::StartupData SerializeNodeContextData(v8::Local<v8::Context> holder,
+                                         int index,
+                                         void* env);
 void DeserializeNodeInternalFields(v8::Local<v8::Object> holder,
                                    int index,
                                    v8::StartupData payload,
                                    void* env);
+void DeserializeNodeContextData(v8::Local<v8::Context> holder,
+                                int index,
+                                v8::StartupData payload,
+                                void* env);
 void SerializeSnapshotableObjects(Realm* realm,
                                   v8::SnapshotCreator* creator,
                                   RealmSerializeInfo* info);

test/parallel/test-snapshot-reproducible.js

@@ -0,0 +1,53 @@
+'use strict';
+
+require('../common');
+const { spawnSyncAndExitWithoutError } = require('../common/child_process');
+const tmpdir = require('../common/tmpdir');
+const fs = require('fs');
+const assert = require('assert');
+const fixtures = require('../common/fixtures');
+
+function generateSnapshot() {
+  tmpdir.refresh();
+
+  spawnSyncAndExitWithoutError(
+    process.execPath,
+    [
+      '--random_seed=42',
+      '--predictable',
+      '--build-snapshot',
+      'node:generate_default_snapshot',
+    ],
+    {
+      cwd: tmpdir.path
+    }
+  );
+  const blobPath = tmpdir.resolve('snapshot.blob');
+  return fs.readFileSync(blobPath);
+}
+
+const buf1 = generateSnapshot();
+const buf2 = generateSnapshot();
+const diff = [];
+let offset = 0;
+const step = 16;
+do {
+  const length = Math.min(buf1.length - offset, step);
+  const slice1 = buf1.slice(offset, offset + length).toString('hex');
+  const slice2 = buf2.slice(offset, offset + length).toString('hex');
+  if (slice1 != slice2) {
+    diff.push({offset, slice1, slice2});
+  }
+  offset += length;
+} while (offset < buf1.length);
+
+assert.strictEqual(offset, buf1.length);
+if (offset < buf2.length) {
+  const length = Math.min(buf2.length - offset, step);
+  const slice2 = buf2.slice(offset, offset + length).toString('hex');
+  diff.push({offset, slice1: '', slice2});
+  offset += length;
+} while (offset < buf2.length);
+
+assert.deepStrictEqual(diff, [], 'Built-in snapshot should not change in different builds.');
+assert.strictEqual(buf1.length, buf2.length);