Skip to content

Commit 7d5e06e

Browse files
brendankennybrendankenny
committed
add streaming trace writer in saveAssets
1 parent fadaef9 commit 7d5e06e

File tree

3 files changed

+171
-25
lines changed

3 files changed

+171
-25
lines changed

‎lighthouse-core/gather/driver.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ const _uniq = arr => Array.from(new Set(arr));
2525

2626
class Driver {
2727
static get MAX_WAIT_FOR_FULLY_LOADED() {
28-
return 30 * 1000;
28+
return 60 * 1000;
2929
}
3030

3131
/**

‎lighthouse-core/lib/asset-saver.js

Lines changed: 76 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
const fs = require('fs');
99
const log = require('lighthouse-logger');
10+
const stream = require('stream');
1011
const stringifySafe = require('json-stringify-safe');
1112
const Metrics = require('./traces/pwmetrics-events');
1213

@@ -102,6 +103,68 @@ function prepareAssets(artifacts, audits) {
102103
.then(_ => assets);
103104
}
104105

106+
/**
107+
* Generates a JSON representation of traceData line-by-line to avoid OOM due to
108+
* very large traces.
109+
* @param {{traceEvents: !Array}} traceData
110+
* @return {!Iterator<string>}
111+
*/
112+
function* traceJsonGenerator(traceData) {
113+
const keys = Object.keys(traceData);
114+
115+
yield '{\n';
116+
117+
// Stringify and emit trace events separately to avoid a giant string in memory.
118+
yield '"traceEvents": [\n';
119+
if (traceData.traceEvents.length > 0) {
120+
const eventsIterator = traceData.traceEvents[Symbol.iterator]();
121+
// Emit first item manually to avoid a trailing comma.
122+
const firstEvent = eventsIterator.next().value;
123+
yield ` ${JSON.stringify(firstEvent)}`;
124+
for (const event of eventsIterator) {
125+
yield `,\n ${JSON.stringify(event)}`;
126+
}
127+
}
128+
yield '\n]';
129+
130+
// Emit the rest of the object (usually just `metadata`)
131+
if (keys.length > 1) {
132+
for (const key of keys) {
133+
if (key === 'traceEvents') continue;
134+
135+
yield `,\n"${key}": ${JSON.stringify(traceData[key], null, 2)}`;
136+
}
137+
}
138+
139+
yield '}\n';
140+
}
141+
142+
/**
143+
* Save a trace as JSON by streaming to disk at traceFilename.
144+
* @param {{traceEvents: !Array}} traceData
145+
* @param {string} traceFilename
146+
* @return {!Promise<undefined>}
147+
*/
148+
function saveTrace(traceData, traceFilename) {
149+
return new Promise((resolve, reject) => {
150+
const traceIter = traceJsonGenerator(traceData);
151+
// A stream that pulls in the next traceJsonGenerator chunk as writeStream
152+
// reads from it. Closes stream with null when iteration is complete.
153+
const traceStream = new stream.Readable({
154+
read() {
155+
const next = traceIter.next();
156+
this.push(next.done ? null : next.value);
157+
}
158+
});
159+
160+
const writeStream = fs.createWriteStream(traceFilename);
161+
writeStream.on('finish', resolve);
162+
writeStream.on('error', reject);
163+
164+
traceStream.pipe(writeStream);
165+
});
166+
}
167+
105168
/**
106169
* Writes trace(s) and associated screenshot(s) to disk.
107170
* @param {!Artifacts} artifacts
@@ -111,28 +174,31 @@ function prepareAssets(artifacts, audits) {
111174
*/
112175
function saveAssets(artifacts, audits, pathWithBasename) {
113176
return prepareAssets(artifacts, audits).then(assets => {
114-
assets.forEach((data, index) => {
115-
const traceFilename = `${pathWithBasename}-${index}.trace.json`;
116-
fs.writeFileSync(traceFilename, JSON.stringify(data.traceData, null, 2));
117-
log.log('trace file saved to disk', traceFilename);
118-
177+
return Promise.all(assets.map((data, index) => {
119178
const devtoolsLogFilename = `${pathWithBasename}-${index}.devtoolslog.json`;
120179
fs.writeFileSync(devtoolsLogFilename, JSON.stringify(data.devtoolsLog, null, 2));
121-
log.log('devtools log saved to disk', devtoolsLogFilename);
180+
log.log('saveAssets', 'devtools log saved to disk: ' + devtoolsLogFilename);
122181

123182
const screenshotsHTMLFilename = `${pathWithBasename}-${index}.screenshots.html`;
124183
fs.writeFileSync(screenshotsHTMLFilename, data.screenshotsHTML);
125-
log.log('screenshots saved to disk', screenshotsHTMLFilename);
184+
log.log('saveAssets', 'screenshots saved to disk: ' + screenshotsHTMLFilename);
126185

127186
const screenshotsJSONFilename = `${pathWithBasename}-${index}.screenshots.json`;
128187
fs.writeFileSync(screenshotsJSONFilename, JSON.stringify(data.screenshots, null, 2));
129-
log.log('screenshots saved to disk', screenshotsJSONFilename);
130-
});
188+
log.log('saveAssets', 'screenshots saved to disk: ' + screenshotsJSONFilename);
189+
190+
const streamTraceFilename = `${pathWithBasename}-${index}.trace.json`;
191+
log.log('saveAssets', 'streaming trace file to disk: ' + streamTraceFilename);
192+
return saveTrace(data.traceData, streamTraceFilename).then(_ => {
193+
log.log('saveAssets', 'trace file streamed to disk: ' + streamTraceFilename);
194+
});
195+
}));
131196
});
132197
}
133198

134199
module.exports = {
135200
saveArtifacts,
136201
saveAssets,
137-
prepareAssets
202+
prepareAssets,
203+
saveTrace
138204
};

‎lighthouse-core/test/lib/asset-saver-test.js

Lines changed: 94 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const traceEvents = require('../fixtures/traces/progressive-app.json');
1515
const dbwTrace = require('../fixtures/traces/dbw_tester.json');
1616
const dbwResults = require('../fixtures/dbw_tester-perf-results.json');
1717
const Audit = require('../../audits/audit.js');
18+
const fullTraceObj = require('../fixtures/traces/progressive-app-m60.json');
1819

1920
/* eslint-env mocha */
2021
describe('asset-saver helper', () => {
@@ -34,24 +35,26 @@ describe('asset-saver helper', () => {
3435
});
3536

3637
describe('saves files', function() {
37-
const artifacts = {
38-
devtoolsLogs: {
39-
[Audit.DEFAULT_PASS]: [{message: 'first'}, {message: 'second'}]
40-
},
41-
traces: {
42-
[Audit.DEFAULT_PASS]: {
43-
traceEvents
44-
}
45-
},
46-
requestScreenshots: () => Promise.resolve(screenshotFilmstrip)
47-
};
38+
before(() => {
39+
const artifacts = {
40+
devtoolsLogs: {
41+
[Audit.DEFAULT_PASS]: [{message: 'first'}, {message: 'second'}]
42+
},
43+
traces: {
44+
[Audit.DEFAULT_PASS]: {
45+
traceEvents
46+
}
47+
},
48+
requestScreenshots: () => Promise.resolve(screenshotFilmstrip)
49+
};
4850

49-
assetSaver.saveAssets(artifacts, dbwResults.audits, process.cwd() + '/the_file');
51+
return assetSaver.saveAssets(artifacts, dbwResults.audits, process.cwd() + '/the_file');
52+
});
5053

51-
it('trace file saved to disk with data', () => {
54+
it('trace file saved to disk with only trace events', () => {
5255
const traceFilename = 'the_file-0.trace.json';
5356
const traceFileContents = fs.readFileSync(traceFilename, 'utf8');
54-
assert.ok(traceFileContents.length > 3000000);
57+
assert.deepStrictEqual(JSON.parse(traceFileContents), {traceEvents});
5558
fs.unlinkSync(traceFilename);
5659
});
5760

@@ -97,4 +100,81 @@ describe('asset-saver helper', () => {
97100
});
98101
});
99102
});
103+
104+
describe('saveTrace', () => {
105+
const traceFilename = 'test-trace-0.json';
106+
107+
afterEach(() => {
108+
fs.unlinkSync(traceFilename);
109+
});
110+
111+
it('correctly saves a trace with metadata to disk', () => {
112+
return assetSaver.saveTrace(fullTraceObj, traceFilename)
113+
.then(_ => {
114+
const traceFileContents = fs.readFileSync(traceFilename, 'utf8');
115+
assert.deepStrictEqual(JSON.parse(traceFileContents), fullTraceObj);
116+
});
117+
});
118+
119+
it('correctly saves a trace with no trace events to disk', () => {
120+
const trace = {
121+
traceEvents: [],
122+
metadata: {
123+
'clock-domain': 'MAC_MACH_ABSOLUTE_TIME',
124+
'cpu-family': 6,
125+
'cpu-model': 70,
126+
'cpu-stepping': 1,
127+
'field-trials': [],
128+
}
129+
};
130+
131+
return assetSaver.saveTrace(trace, traceFilename)
132+
.then(_ => {
133+
const traceFileContents = fs.readFileSync(traceFilename, 'utf8');
134+
assert.deepStrictEqual(JSON.parse(traceFileContents), trace);
135+
});
136+
});
137+
138+
it('correctly saves a trace with multiple extra properties to disk', () => {
139+
const trace = {
140+
traceEvents,
141+
metadata: fullTraceObj.metadata,
142+
someProp: 555,
143+
anotherProp: {
144+
unlikely: {
145+
nested: [
146+
'value'
147+
]
148+
}
149+
},
150+
};
151+
152+
return assetSaver.saveTrace(trace, traceFilename)
153+
.then(_ => {
154+
const traceFileContents = fs.readFileSync(traceFilename, 'utf8');
155+
assert.deepStrictEqual(JSON.parse(traceFileContents), trace);
156+
});
157+
});
158+
159+
it('can save traces over 256MB (slow)', () => {
160+
// Create a trace that wil be longer than 256MB when stringified, the hard
161+
// limit of a string in v8.
162+
// https://mobile.twitter.com/bmeurer/status/879276976523157505
163+
const baseEventsLength = JSON.stringify(traceEvents).length;
164+
const countNeeded = Math.ceil(Math.pow(2, 28) / baseEventsLength);
165+
let longTraceEvents = [];
166+
for (let i = 0; i < countNeeded; i++) {
167+
longTraceEvents = longTraceEvents.concat(traceEvents);
168+
}
169+
const trace = {
170+
traceEvents: longTraceEvents
171+
};
172+
173+
return assetSaver.saveTrace(trace, traceFilename)
174+
.then(_ => {
175+
const fileStats = fs.lstatSync(traceFilename);
176+
assert.ok(fileStats.size > Math.pow(2, 28));
177+
});
178+
});
179+
});
100180
});

0 commit comments

Comments
 (0)