add streaming json parser

Author: paulirish

lighthouse-core/gather/driver.js

@@ -10,6 +10,7 @@ const emulation = require('../lib/emulation');
 const Element = require('../lib/element');
 const EventEmitter = require('events').EventEmitter;
 const URL = require('../lib/url-shim');
+const TraceParser = require('../lib/traces/trace-parser');
 
 const log = require('lighthouse-logger');
 const DevtoolsLog = require('./devtools-log');
@@ -25,7 +26,7 @@ const _uniq = arr => Array.from(new Set(arr));
 
 class Driver {
   static get MAX_WAIT_FOR_FULLY_LOADED() {
-    return 60 * 1000;
+    return 30 * 1000;
   }
 
   /**
@@ -801,7 +802,7 @@ class Driver {
   _readTraceFromStream(streamHandle) {
     return new Promise((resolve, reject) => {
       let isEOF = false;
-      let result = '';
+      const parser = new TraceParser();
 
       const readArguments = {
         handle: streamHandle.stream
@@ -812,11 +813,11 @@ class Driver {
           return;
         }
 
-        result += response.data;
+        parser.parseChunk(response.data);
 
         if (response.eof) {
           isEOF = true;
-          return resolve(JSON.parse(result));
+          return resolve(parser.getTrace());
         }
 
         return this.sendCommand('IO.read', readArguments).then(onChunkRead);

lighthouse-core/lib/traces/trace-parser.js

@@ -0,0 +1,70 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const WebInspector = require('../web-inspector');
+
+/**
+ * Traces > 256MB hit limits in V8, so TraceParser will parse the trace events stream as it's
+ * received. We use DevTools' TimelineLoader for the heavy lifting, as it has a fast trace-specific
+ * streaming JSON parser.
+ * The resulting trace doesn't include the "metadata" property, as it's excluded via DevTools'
+ * implementation.
+ */
+class TraceParser {
+  constructor() {
+    this.traceEvents = [];
+
+    this.tracingModel = {
+      reset: _ => this._reset(),
+      addEvents: evts => this._addEvents(evts),
+    };
+
+    const delegateMock = {
+      loadingProgress: _ => {},
+      loadingStarted: _ => {},
+      loadingComplete: success => {
+        if (!success) throw new Error('Parsing problem');
+      }
+    };
+    this.loader = new WebInspector.TimelineLoader(this.tracingModel, delegateMock);
+  }
+
+  /**
+   * Reset the trace events array
+   */
+  _reset() {
+    this.traceEvents = [];
+  }
+
+  /**
+   * Adds parsed trace events to array
+   * @param {!Array<!TraceEvent>} evts
+   */
+  _addEvents(evts) {
+    this.traceEvents.push(...evts);
+  }
+
+  /**
+   * Receive chunk of streamed trace
+   * @param {string} data
+   */
+  parseChunk(data) {
+    this.loader.write(data);
+  }
+
+  /**
+   * Returns entire trace
+   * @return {{traceEvents: !Array<!TraceEvent>}}
+   */
+  getTrace() {
+    return {
+      traceEvents: this.traceEvents
+    };
+  }
+}
+
+module.exports = TraceParser;

lighthouse-core/lib/web-inspector.js

@@ -176,6 +176,11 @@ module.exports = (function() {
   require('chrome-devtools-frontend/front_end/timeline_model/TimelineModel.js');
   require('chrome-devtools-frontend/front_end/ui_lazy/SortableDataGrid.js');
   require('chrome-devtools-frontend/front_end/timeline/TimelineTreeView.js');
+
+  // used for streaming json parsing
+  require('chrome-devtools-frontend/front_end/common/TextUtils.js');
+  require('chrome-devtools-frontend/front_end/timeline/TimelineLoader.js');
+
   require('chrome-devtools-frontend/front_end/timeline_model/TimelineProfileTree.js');
   require('chrome-devtools-frontend/front_end/components_lazy/FilmStripModel.js');
   require('chrome-devtools-frontend/front_end/timeline_model/TimelineIRModel.js');

lighthouse-core/test/lib/traces/trace-parser-test.js

@@ -0,0 +1,90 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const TraceParser = require('../../../lib/traces/trace-parser');
+const fs = require('fs');
+const assert = require('assert');
+
+
+/* eslint-env mocha */
+describe('traceParser parser', () => {
+  it('returns preact trace data the same as JSON.parse', (done) => {
+    const filename = '../../fixtures/traces/progressive-app-m60.json';
+    const readStream = fs.createReadStream(__dirname + '/' + filename, {
+      encoding: 'utf-8',
+      // devtools sends traces in 10mb chunks, but this trace is 12MB so we'll do a few chunks
+      highWaterMark: 4 * 1024 * 1024
+    });
+    const parser = new TraceParser();
+
+    readStream.on('data', (chunk) => {
+      parser.parseChunk(chunk);
+    });
+    readStream.on('end', () => {
+      const streamedTrace = parser.getTrace();
+      const readTrace = require(filename);
+
+      assert.equal(streamedTrace.traceEvents.length, readTrace.traceEvents.length);
+      assert.deepStrictEqual(streamedTrace.traceEvents, readTrace.traceEvents);
+
+      done();
+    });
+  });
+
+
+  it('parses a trace > 256mb (slow)', () => {
+    const parser = new TraceParser();
+    let bytesRead = 0;
+    // FYI: this trace doesn't have a traceEvents property ;)
+    const events = require('../../fixtures/traces/devtools-homepage-w-screenshots-trace.json');
+
+    /**
+     * This function will synthesize a trace that's over 256 MB. To do that, we'll take an existing
+     * trace and repeat the same events again and again until we've gone over 256 MB.
+     * Note: We repeat all but the last event, as it's the CpuProfile event, and it triggers
+     * specific handling in the devtools streaming parser.
+     * Once we reach > 256 MB, we add in the CpuProfile event.
+     */
+    function buildAndParse256mbTrace() {
+      const stripOuterBrackets = str => str.replace(/^\[/, '').replace(/\]$/, '');
+      const partialEventsStr = events => stripOuterBrackets(JSON.stringify(events));
+      const traceEventsStr = partialEventsStr(events.slice(0, events.length-2)) + ',';
+
+      // read the trace intro
+      parser.parseChunk(`{"traceEvents": [${traceEventsStr}`);
+      bytesRead += traceEventsStr.length;
+
+      // just keep reading until we've gone over 256 MB
+      // 256 MB is hard limit of a string in v8
+      // https://mobile.twitter.com/bmeurer/status/879276976523157505
+      while (bytesRead <= (Math.pow(2, 28)) - 16) {
+        parser.parseChunk(traceEventsStr);
+        bytesRead += traceEventsStr.length;
+      }
+
+      // the CPU Profiler event is last (and big), inject it just once
+      const lastEventStr = partialEventsStr(events.slice(-1));
+      parser.parseChunk(lastEventStr + ']}');
+      bytesRead += lastEventStr.length;
+    }
+
+    buildAndParse256mbTrace();
+    const streamedTrace = parser.getTrace();
+
+    assert.ok(bytesRead > 256 * 1024 * 1024, `${bytesRead} bytes read`);
+    assert.strictEqual(bytesRead, 270179102, `${bytesRead} bytes read`);
+
+    // if > 256 MB are read we should have ~480,000 trace events
+    assert.ok(streamedTrace.traceEvents.length > 400 * 1000, 'not >400,000 trace events');
+    assert.ok(streamedTrace.traceEvents.length > events.length * 5, 'not way more trace events');
+    assert.strictEqual(streamedTrace.traceEvents.length, 480151);
+
+    assert.deepStrictEqual(
+        streamedTrace.traceEvents[events.length - 2],
+        events[0]);
+  });
+});