diff --git a/test/browsercontext.spec.js b/test/browsercontext.spec.js index 71ce91ca226ed..2fe288de4fb47 100644 --- a/test/browsercontext.spec.js +++ b/test/browsercontext.spec.js @@ -16,7 +16,7 @@ */ const utils = require('./utils'); -const {FFOX, CHROMIUM, WEBKIT, MAC} = utils.testOptions(browserType); +const {FFOX, CHROMIUM, WEBKIT, MAC, CHANNEL} = utils.testOptions(browserType); describe('BrowserContext', function() { it('should create new context', async function({browser}) { @@ -121,7 +121,7 @@ describe('BrowserContext', function() { let error = await promise; expect(error.message).toContain('Context closed'); }); - it('close() should be callable twice', async({browser}) => { + it.fail(CHANNEL)('close() should be callable twice', async({browser}) => { const context = await browser.newContext(); await Promise.all([ context.close(), diff --git a/test/environments.js b/test/environments.js deleted file mode 100644 index 71b2844e3fb30..0000000000000 --- a/test/environments.js +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright 2019 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const utils = require('./utils'); -const fs = require('fs'); -const path = require('path'); -const rm = require('rimraf').sync; -const {TestServer} = require('../utils/testserver/'); -const { DispatcherConnection } = require('../lib/rpc/server/dispatcher'); -const { Connection } = require('../lib/rpc/client/connection'); -const { BrowserTypeDispatcher } = require('../lib/rpc/server/browserTypeDispatcher'); - -class ServerEnvironment { - async beforeAll(state) { - const assetsPath = path.join(__dirname, 'assets'); - const cachedPath = path.join(__dirname, 'assets', 'cached'); - - const port = 8907 + state.parallelIndex * 2; - state.server = await TestServer.create(assetsPath, port); - state.server.enableHTTPCache(cachedPath); - state.server.PORT = port; - state.server.PREFIX = `http://localhost:${port}`; - state.server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`; - state.server.EMPTY_PAGE = `http://localhost:${port}/empty.html`; - - const httpsPort = port + 1; - state.httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort); - state.httpsServer.enableHTTPCache(cachedPath); - state.httpsServer.PORT = httpsPort; - state.httpsServer.PREFIX = `https://localhost:${httpsPort}`; - state.httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`; - state.httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`; - } - - async afterAll({server, httpsServer}) { - await Promise.all([ - server.stop(), - httpsServer.stop(), - ]); - } - - async beforeEach(state) { - state.server.reset(); - state.httpsServer.reset(); - } -} - -class DefaultBrowserOptionsEnvironment { - constructor(defaultBrowserOptions, dumpLogOnFailure, playwrightPath) { - this._defaultBrowserOptions = defaultBrowserOptions; - this._dumpLogOnFailure = dumpLogOnFailure; - this._playwrightPath = playwrightPath; - this._loggerSymbol = Symbol('DefaultBrowserOptionsEnvironment.logger'); - } - - async beforeAll(state) { - state[this._loggerSymbol] = utils.createTestLogger(this._dumpLogOnFailure, null, 'extra'); - state.defaultBrowserOptions = { - ...this._defaultBrowserOptions, - logger: state[this._loggerSymbol], - }; - state.playwrightPath = this._playwrightPath; - } - - async beforeEach(state, testRun) { - state[this._loggerSymbol].setTestRun(testRun); - } - - async afterEach(state) { - state[this._loggerSymbol].setTestRun(null); - } -} - -// simulate globalSetup per browserType that happens only once regardless of TestWorker. -const hasBeenCleaned = new Set(); - -class GoldenEnvironment { - async beforeAll(state) { - const { OUTPUT_DIR, GOLDEN_DIR } = utils.testOptions(state.browserType); - if (!hasBeenCleaned.has(state.browserType)) { - hasBeenCleaned.add(state.browserType); - if (fs.existsSync(OUTPUT_DIR)) - rm(OUTPUT_DIR); - fs.mkdirSync(OUTPUT_DIR, { recursive: true }); - } - state.golden = goldenName => ({ goldenPath: GOLDEN_DIR, outputPath: OUTPUT_DIR, goldenName }); - } - - async afterAll(state) { - delete state.golden; - } - - async afterEach(state, testRun) { - if (state.browser && state.browser.contexts().length !== 0) { - if (testRun.ok()) - console.warn(`\nWARNING: test "${testRun.test().fullName()}" (${testRun.test().location()}) did not close all created contexts!\n`); - await Promise.all(state.browser.contexts().map(context => context.close())); - } - } -} - -class TraceTestEnvironment { - static enableForTest(test) { - test.setTimeout(100000000); - test.addEnvironment(new TraceTestEnvironment()); - } - - constructor() { - this._session = null; - } - - async beforeEach() { - const inspector = require('inspector'); - const fs = require('fs'); - const util = require('util'); - const url = require('url'); - const readFileAsync = util.promisify(fs.readFile.bind(fs)); - this._session = new inspector.Session(); - this._session.connect(); - const postAsync = util.promisify(this._session.post.bind(this._session)); - await postAsync('Debugger.enable'); - const setBreakpointCommands = []; - const N = t.body().toString().split('\n').length; - const location = t.location(); - const lines = (await readFileAsync(location.filePath(), 'utf8')).split('\n'); - for (let line = 0; line < N; ++line) { - const lineNumber = line + location.lineNumber(); - setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', { - url: url.pathToFileURL(location.filePath()), - lineNumber, - condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`, - }).catch(e => {})); - } - await Promise.all(setBreakpointCommands); - } - - async afterEach() { - this._session.disconnect(); - } -} - -class PlaywrightEnvironment { - constructor(playwright) { - this._playwright = playwright; - } - - name() { return 'Playwright'; }; - beforeAll(state) { state.playwright = this._playwright; } - afterAll(state) { delete state.playwright; } -} - -class BrowserTypeEnvironment { - constructor(browserType) { - this._browserType = browserType; - } - - async beforeAll(state) { - // Channel substitute - let overridenBrowserType = this._browserType; - if (process.env.PWCHANNEL) { - const dispatcherConnection = new DispatcherConnection(); - const connection = new Connection(); - dispatcherConnection.onmessage = async message => { - setImmediate(() => connection.dispatch(message)); - }; - connection.onmessage = async message => { - const result = await dispatcherConnection.dispatch(message); - await new Promise(f => setImmediate(f)); - return result; - }; - new BrowserTypeDispatcher(dispatcherConnection.createScope(), this._browserType); - overridenBrowserType = await connection.waitForObjectWithKnownName(this._browserType.name()); - } - state.browserType = overridenBrowserType; - } - - async afterAll(state) { - delete state.browserType; - } -} - -class BrowserEnvironment { - constructor(browserType, launchOptions, dumpLogOnFailure) { - this._browserType = browserType; - this._launchOptions = launchOptions; - this._dumpLogOnFailure = dumpLogOnFailure; - this._loggerSymbol = Symbol('BrowserEnvironment.logger'); - } - - name() { return this._browserType.name(); } - - async beforeAll(state) { - state[this._loggerSymbol] = utils.createTestLogger(this._dumpLogOnFailure); - state.browser = await this._browserType.launch({ - ...this._launchOptions, - logger: state[this._loggerSymbol], - }); - } - - async afterAll(state) { - await state.browser.close(); - delete state.browser; - } - - async beforeEach(state, testRun) { - state[this._loggerSymbol].setTestRun(testRun); - } - - async afterEach(state, testRun) { - state[this._loggerSymbol].setTestRun(null); - } -} - -class PageEnvironment { - async beforeEach(state) { - state.context = await state.browser.newContext(); - state.page = await state.context.newPage(); - } - - async afterEach(state) { - await state.context.close(); - state.context = null; - state.page = null; - } -} - -module.exports = { - ServerEnvironment, - GoldenEnvironment, - TraceTestEnvironment, - DefaultBrowserOptionsEnvironment, - PlaywrightEnvironment, - BrowserTypeEnvironment, - BrowserEnvironment, - PageEnvironment, -}; diff --git a/test/page.spec.js b/test/page.spec.js index 1758009ea5b1b..419dd64cc5eb3 100644 --- a/test/page.spec.js +++ b/test/page.spec.js @@ -18,7 +18,7 @@ const path = require('path'); const util = require('util'); const vm = require('vm'); -const {FFOX, CHROMIUM, WEBKIT, WIN} = require('./utils').testOptions(browserType); +const {FFOX, CHROMIUM, WEBKIT, WIN, CHANNEL} = require('./utils').testOptions(browserType); describe('Page.close', function() { it('should reject all promises when page is closed', async({context}) => { @@ -154,7 +154,7 @@ describe.fail(FFOX && WIN)('Page.Events.Crash', function() { const error = await promise; expect(error.message).toContain('Navigation failed because page crashed'); }); - it('should be able to close context when page crashes', async({page}) => { + it.fail(CHANNEL)('should be able to close context when page crashes', async({page}) => { await page.setContent(`
This page should crash
`); crash(page); await page.waitForEvent('crash'); @@ -1306,3 +1306,8 @@ describe('Page api coverage', function() { }); }); +describe.skip(!CHANNEL)('Page channel', function() { + it('page should be client stub', async({page, server}) => { + expect(!!page._channel).toBeTruthy(); + }); +}); diff --git a/test/test.config.js b/test/test.config.js index 016578aaad5ce..924d44f7d1c24 100644 --- a/test/test.config.js +++ b/test/test.config.js @@ -15,21 +15,85 @@ * limitations under the License. */ +const fs = require('fs'); const path = require('path'); +const rm = require('rimraf').sync; const utils = require('./utils'); -const {DefaultBrowserOptionsEnvironment, ServerEnvironment, GoldenEnvironment, TraceTestEnvironment} = require('./environments.js'); +const {TestServer} = require('../utils/testserver/'); +const {Environment} = require('../utils/testrunner/Test'); const playwrightPath = path.join(__dirname, '..'); -const dumpLogOnFailure = valueFromEnv('DEBUGP', false); -const defaultBrowserOptionsEnvironment = new DefaultBrowserOptionsEnvironment({ - handleSIGINT: false, - slowMo: valueFromEnv('SLOW_MO', 0), - headless: !!valueFromEnv('HEADLESS', true), -}, dumpLogOnFailure, playwrightPath); +const serverEnvironment = new Environment('TestServer'); +serverEnvironment.beforeAll(async state => { + const assetsPath = path.join(__dirname, 'assets'); + const cachedPath = path.join(__dirname, 'assets', 'cached'); -const serverEnvironment = new ServerEnvironment(); -const customEnvironment = new GoldenEnvironment(); + const port = 8907 + state.parallelIndex * 2; + state.server = await TestServer.create(assetsPath, port); + state.server.enableHTTPCache(cachedPath); + state.server.PORT = port; + state.server.PREFIX = `http://localhost:${port}`; + state.server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`; + state.server.EMPTY_PAGE = `http://localhost:${port}/empty.html`; + + const httpsPort = port + 1; + state.httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort); + state.httpsServer.enableHTTPCache(cachedPath); + state.httpsServer.PORT = httpsPort; + state.httpsServer.PREFIX = `https://localhost:${httpsPort}`; + state.httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`; + state.httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`; + + state._extraLogger = utils.createTestLogger(valueFromEnv('DEBUGP', false), null, 'extra'); + state.defaultBrowserOptions = { + handleSIGINT: false, + slowMo: valueFromEnv('SLOW_MO', 0), + headless: !!valueFromEnv('HEADLESS', true), + logger: state._extraLogger, + }; + state.playwrightPath = playwrightPath; +}); +serverEnvironment.afterAll(async({server, httpsServer}) => { + await Promise.all([ + server.stop(), + httpsServer.stop(), + ]); +}); +serverEnvironment.beforeEach(async(state, testRun) => { + state.server.reset(); + state.httpsServer.reset(); + state._extraLogger.setTestRun(testRun); +}); +serverEnvironment.afterEach(async(state) => { + state._extraLogger.setTestRun(null); +}); + +const customEnvironment = new Environment('Golden+CheckContexts'); + +// simulate globalSetup per browserType that happens only once regardless of TestWorker. +const hasBeenCleaned = new Set(); + +customEnvironment.beforeAll(async state => { + const { OUTPUT_DIR, GOLDEN_DIR } = require('./utils').testOptions(state.browserType); + if (!hasBeenCleaned.has(state.browserType)) { + hasBeenCleaned.add(state.browserType); + if (fs.existsSync(OUTPUT_DIR)) + rm(OUTPUT_DIR); + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + } + state.golden = goldenName => ({ goldenPath: GOLDEN_DIR, outputPath: OUTPUT_DIR, goldenName }); +}); +customEnvironment.afterAll(async state => { + delete state.golden; +}); +customEnvironment.afterEach(async (state, testRun) => { + if (state.browser && state.browser.contexts().length !== 0) { + if (testRun.ok()) + console.warn(`\nWARNING: test "${testRun.test().fullName()}" (${testRun.test().location()}) did not close all created contexts!\n`); + await Promise.all(state.browser.contexts().map(context => context.close())); + } +}); function valueFromEnv(name, defaultValue) { if (!(name in process.env)) @@ -44,7 +108,39 @@ function setupTestRunner(testRunner) { collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3)); - collector.addTestAttribute('debug', t => TraceTestEnvironment.enableForTest(t)); + collector.addTestAttribute('debug', t => { + t.setTimeout(100000000); + + let session; + t.environment().beforeEach(async () => { + const inspector = require('inspector'); + const fs = require('fs'); + const util = require('util'); + const url = require('url'); + const readFileAsync = util.promisify(fs.readFile.bind(fs)); + session = new inspector.Session(); + session.connect(); + const postAsync = util.promisify(session.post.bind(session)); + await postAsync('Debugger.enable'); + const setBreakpointCommands = []; + const N = t.body().toString().split('\n').length; + const location = t.location(); + const lines = (await readFileAsync(location.filePath(), 'utf8')).split('\n'); + for (let line = 0; line < N; ++line) { + const lineNumber = line + location.lineNumber(); + setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', { + url: url.pathToFileURL(location.filePath()), + lineNumber, + condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`, + }).catch(e => {})); + } + await Promise.all(setBreakpointCommands); + }); + + t.environment().afterEach(async () => { + session.disconnect(); + }); + }); testRunner.api().fdescribe = testRunner.api().describe.only; testRunner.api().xdescribe = testRunner.api().describe.skip(true); testRunner.api().fit = testRunner.api().it.only; @@ -65,7 +161,7 @@ module.exports = { headless: !!valueFromEnv('HEADLESS', true), }, - globalEnvironments: [defaultBrowserOptionsEnvironment, serverEnvironment], + globalEnvironments: [serverEnvironment], setupTestRunner, specs: [ diff --git a/test/test.js b/test/test.js index 87af708b67561..c79c7eb0f428f 100644 --- a/test/test.js +++ b/test/test.js @@ -18,7 +18,10 @@ const fs = require('fs'); const utils = require('./utils'); const TestRunner = require('../utils/testrunner/'); -const { PlaywrightEnvironment, BrowserTypeEnvironment, BrowserEnvironment, PageEnvironment} = require('./environments.js'); +const { Environment } = require('../utils/testrunner/Test'); +const { DispatcherConnection } = require('../lib/rpc/server/dispatcher'); +const { Connection } = require('../lib/rpc/client/connection'); +const { BrowserTypeDispatcher } = require('../lib/rpc/server/browserTypeDispatcher'); Error.stackTraceLimit = 15; @@ -79,7 +82,15 @@ function collect(browserNames) { const { setUnderTest } = require(require('path').join(playwrightPath, 'lib/helper.js')); setUnderTest(); - testRunner.collector().useEnvironment(new PlaywrightEnvironment(playwright)); + const playwrightEnvironment = new Environment('Playwright'); + playwrightEnvironment.beforeAll(async state => { + state.playwright = playwright; + }); + playwrightEnvironment.afterAll(async state => { + delete state.playwright; + }); + + testRunner.collector().useEnvironment(playwrightEnvironment); for (const e of config.globalEnvironments || []) testRunner.collector().useEnvironment(e); @@ -87,7 +98,30 @@ function collect(browserNames) { for (const browserName of browserNames) { const browserType = playwright[browserName]; - const browserTypeEnvironment = new BrowserTypeEnvironment(browserType); + + const browserTypeEnvironment = new Environment('BrowserType'); + browserTypeEnvironment.beforeAll(async state => { + // Channel substitute + let overridenBrowserType = browserType; + if (process.env.PWCHANNEL) { + const dispatcherConnection = new DispatcherConnection(); + const connection = new Connection(); + dispatcherConnection.onmessage = async message => { + setImmediate(() => connection.dispatch(message)); + }; + connection.onmessage = async message => { + const result = await dispatcherConnection.dispatch(message); + await new Promise(f => setImmediate(f)); + return result; + }; + new BrowserTypeDispatcher(dispatcherConnection.createScope(), browserType); + overridenBrowserType = await connection.waitForObjectWithKnownName(browserType.name()); + } + state.browserType = overridenBrowserType; + }); + browserTypeEnvironment.afterAll(async state => { + delete state.browserType; + }); // TODO: maybe launch options per browser? const launchOptions = { @@ -107,8 +141,33 @@ function collect(browserNames) { throw new Error(`Browser is not downloaded. Run 'npm install' and try to re-run tests`); } - const browserEnvironment = new BrowserEnvironment(browserType, launchOptions, config.dumpLogOnFailure); - const pageEnvironment = new PageEnvironment(); + const browserEnvironment = new Environment(browserName); + browserEnvironment.beforeAll(async state => { + state._logger = utils.createTestLogger(config.dumpLogOnFailure); + state.browser = await state.browserType.launch({...launchOptions, logger: state._logger}); + }); + browserEnvironment.afterAll(async state => { + await state.browser.close(); + delete state.browser; + delete state._logger; + }); + browserEnvironment.beforeEach(async(state, testRun) => { + state._logger.setTestRun(testRun); + }); + browserEnvironment.afterEach(async (state, testRun) => { + state._logger.setTestRun(null); + }); + + const pageEnvironment = new Environment('Page'); + pageEnvironment.beforeEach(async state => { + state.context = await state.browser.newContext(); + state.page = await state.context.newPage(); + }); + pageEnvironment.afterEach(async state => { + await state.context.close(); + state.context = null; + state.page = null; + }); const suiteName = { 'chromium': 'Chromium', 'firefox': 'Firefox', 'webkit': 'WebKit' }[browserName]; describe(suiteName, () => { diff --git a/utils/testrunner/Test.js b/utils/testrunner/Test.js index d9005a05fd768..e460d5f6d52c0 100644 --- a/utils/testrunner/Test.js +++ b/utils/testrunner/Test.js @@ -22,6 +22,60 @@ const TestExpectation = { Fail: 'fail', }; +function createHook(callback, name) { + const location = Location.getCallerLocation(); + return { name, body: callback, location }; +} + +class Environment { + constructor(name) { + this._name = name; + this._hooks = []; + } + + name() { + return this._name; + } + + beforeEach(callback) { + this._hooks.push(createHook(callback, 'beforeEach')); + return this; + } + + afterEach(callback) { + this._hooks.push(createHook(callback, 'afterEach')); + return this; + } + + beforeAll(callback) { + this._hooks.push(createHook(callback, 'beforeAll')); + return this; + } + + afterAll(callback) { + this._hooks.push(createHook(callback, 'afterAll')); + return this; + } + + globalSetup(callback) { + this._hooks.push(createHook(callback, 'globalSetup')); + return this; + } + + globalTeardown(callback) { + this._hooks.push(createHook(callback, 'globalTeardown')); + return this; + } + + hooks(name) { + return this._hooks.filter(hook => !name || hook.name === name); + } + + isEmpty() { + return !this._hooks.length; + } +} + class Test { constructor(suite, name, callback, location) { this._suite = suite; @@ -32,7 +86,8 @@ class Test { this._body = callback; this._location = location; this._timeout = 100000000; - this._environments = []; + this._defaultEnvironment = new Environment(this._fullName); + this._environments = [this._defaultEnvironment]; this.Expectations = { ...TestExpectation }; } @@ -83,6 +138,10 @@ class Test { return this; } + environment() { + return this._defaultEnvironment; + } + addEnvironment(environment) { this._environments.push(environment); return this; @@ -105,50 +164,49 @@ class Suite { this._location = location; this._skipped = false; this._expectation = TestExpectation.Ok; - - this._defaultEnvironment = { - name() { return this._fullName; }, - }; - + this._defaultEnvironment = new Environment(this._fullName); this._environments = [this._defaultEnvironment]; this.Expectations = { ...TestExpectation }; } - _addHook(name, callback) { - if (this._defaultEnvironment[name]) - throw new Error(`ERROR: cannot re-assign hook "${name}" for suite "${this._fullName}"`); - this._defaultEnvironment[name] = callback; + parentSuite() { + return this._parentSuite; } - beforeEach(callback) { this._addHook('beforeEach', callback); } - afterEach(callback) { this._addHook('afterEach', callback); } - beforeAll(callback) { this._addHook('beforeAll', callback); } - afterAll(callback) { this._addHook('afterAll', callback); } - globalSetup(callback) { this._addHook('globalSetup', callback); } - globalTeardown(callback) { this._addHook('globalTeardown', callback); } - - parentSuite() { return this._parentSuite; } - - name() { return this._name; } + name() { + return this._name; + } - fullName() { return this._fullName; } + fullName() { + return this._fullName; + } - skipped() { return this._skipped; } + skipped() { + return this._skipped; + } setSkipped(skipped) { this._skipped = skipped; return this; } - location() { return this._location; } + location() { + return this._location; + } - expectation() { return this._expectation; } + expectation() { + return this._expectation; + } setExpectation(expectation) { this._expectation = expectation; return this; } + environment() { + return this._defaultEnvironment; + } + addEnvironment(environment) { this._environments.push(environment); return this; @@ -163,4 +221,4 @@ class Suite { } } -module.exports = { TestExpectation, Test, Suite }; +module.exports = { TestExpectation, Environment, Test, Suite }; diff --git a/utils/testrunner/TestCollector.js b/utils/testrunner/TestCollector.js index 7f95eefcf5119..91115b5906942 100644 --- a/utils/testrunner/TestCollector.js +++ b/utils/testrunner/TestCollector.js @@ -195,12 +195,12 @@ class TestCollector { callback(test, ...args); this._tests.push(test); }); - this._api.beforeAll = callback => this._currentSuite.beforeAll(callback); - this._api.beforeEach = callback => this._currentSuite.beforeEach(callback); - this._api.afterAll = callback => this._currentSuite.afterAll(callback); - this._api.afterEach = callback => this._currentSuite.afterEach(callback); - this._api.globalSetup = callback => this._currentSuite.globalSetup(callback); - this._api.globalTeardown = callback => this._currentSuite.globalTeardown(callback); + this._api.beforeAll = callback => this._currentSuite.environment().beforeAll(callback); + this._api.beforeEach = callback => this._currentSuite.environment().beforeEach(callback); + this._api.afterAll = callback => this._currentSuite.environment().afterAll(callback); + this._api.afterEach = callback => this._currentSuite.environment().afterEach(callback); + this._api.globalSetup = callback => this._currentSuite.environment().globalSetup(callback); + this._api.globalTeardown = callback => this._currentSuite.environment().globalTeardown(callback); } useEnvironment(environment) { diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index f790d0231dc8b..4d839cdb91782 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -46,11 +46,6 @@ const TestResult = { Crashed: 'crashed', // If testrunner crashed due to this test }; -function isEmptyEnvironment(env) { - return !env.afterEach && !env.afterAll && !env.beforeEach && !env.beforeAll && - !env.globalSetup && !env.globalTeardown; -} - class TestRun { constructor(test) { this._test = test; @@ -61,9 +56,9 @@ class TestRun { this._workerId = null; this._output = []; - this._environments = test._environments.filter(env => !isEmptyEnvironment(env)).reverse(); + this._environments = test._environments.filter(env => !env.isEmpty()).reverse(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) - this._environments.push(...suite._environments.filter(env => !isEmptyEnvironment(env)).reverse()); + this._environments.push(...suite._environments.filter(env => !env.isEmpty()).reverse()); this._environments.reverse(); } @@ -203,9 +198,9 @@ class TestWorker { if (this._markTerminated(testRun)) return; const environment = this._environmentStack.pop(); - if (!await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, testRun)) + if (!await this._hookRunner.runAfterAll(environment, this, testRun, [this._state])) return; - if (!await this._hookRunner.maybeRunGlobalTeardown(environment)) + if (!await this._hookRunner.ensureGlobalTeardown(environment)) return; } while (this._environmentStack.length < environmentStack.length) { @@ -213,9 +208,9 @@ class TestWorker { return; const environment = environmentStack[this._environmentStack.length]; this._environmentStack.push(environment); - if (!await this._hookRunner.maybeRunGlobalSetup(environment)) + if (!await this._hookRunner.ensureGlobalSetup(environment)) return; - if (!await this._hookRunner.runHook(environment, 'beforeAll', [this._state], this, testRun)) + if (!await this._hookRunner.runBeforeAll(environment, this, testRun, [this._state])) return; } @@ -227,7 +222,7 @@ class TestWorker { await this._willStartTestRun(testRun); for (const environment of this._environmentStack) { - await this._hookRunner.runHook(environment, 'beforeEach', [this._state, testRun], this, testRun); + await this._hookRunner.runBeforeEach(environment, this, testRun, [this._state, testRun]); } if (!testRun._error && !this._markTerminated(testRun)) { @@ -250,7 +245,7 @@ class TestWorker { } for (const environment of this._environmentStack.slice().reverse()) - await this._hookRunner.runHook(environment, 'afterEach', [this._state, testRun], this, testRun); + await this._hookRunner.runAfterEach(environment, this, testRun, [this._state, testRun]); await this._didFinishTestRun(testRun); } @@ -279,8 +274,8 @@ class TestWorker { async shutdown() { while (this._environmentStack.length > 0) { const environment = this._environmentStack.pop(); - await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, null); - await this._hookRunner.maybeRunGlobalTeardown(environment); + await this._hookRunner.runAfterAll(environment, this, null, [this._state]); + await this._hookRunner.ensureGlobalTeardown(environment); } } } @@ -327,7 +322,7 @@ class HookRunner { } } - async _runHookInternal(worker, testRun, hook, fullName, hookArgs = []) { + async _runHook(worker, testRun, hook, fullName, hookArgs = []) { await this._willStartHook(worker, testRun, hook, fullName); const timeout = this._testRunner._hookTimeout; const { promise, terminate } = runUserCallback(hook.body, timeout, hookArgs); @@ -342,7 +337,7 @@ class HookRunner { } let message; if (error === TimeoutError) { - message = `Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`; + message = `${hook.location.toDetailedString()} - Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`; error = null; } else if (error === TerminatedError) { // Do not report termination details - it's just noise. @@ -351,7 +346,7 @@ class HookRunner { } else { if (error.stack) await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); - message = `FAILED while running "${hook.name}" in suite "${fullName}": `; + message = `${hook.location.toDetailedString()} - FAILED while running "${hook.name}" in suite "${fullName}": `; } await this._didFailHook(worker, testRun, hook, fullName, message, error); if (testRun) @@ -363,18 +358,50 @@ class HookRunner { return true; } - async runHook(environment, hookName, hookArgs, worker = null, testRun = null) { - const hookBody = environment[hookName]; - if (!hookBody) - return true; - const envName = environment.name ? environment.name() : environment.constructor.name; - return await this._runHookInternal(worker, testRun, {name: hookName, body: hookBody.bind(environment)}, envName, hookArgs); + async runAfterAll(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('afterAll')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; + } + + async runBeforeAll(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('beforeAll')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; + } + + async runAfterEach(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('afterEach')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; + } + + async runBeforeEach(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('beforeEach')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; } - async maybeRunGlobalSetup(environment) { + async ensureGlobalSetup(environment) { const globalState = this._environmentToGlobalState.get(environment); - if (!globalState.globalSetupPromise) - globalState.globalSetupPromise = this.runHook(environment, 'globalSetup', []); + if (!globalState.globalSetupPromise) { + globalState.globalSetupPromise = (async () => { + let result = true; + for (const hook of environment.hooks('globalSetup')) { + if (!await this._runHook(null /* worker */, null /* testRun */, hook, environment.name(), [])) + result = false; + } + return result; + })(); + } if (!await globalState.globalSetupPromise) { await this._testRunner._terminate(TestResult.Crashed, 'Global setup failed!', false, null); return false; @@ -382,11 +409,19 @@ class HookRunner { return true; } - async maybeRunGlobalTeardown(environment) { + async ensureGlobalTeardown(environment) { const globalState = this._environmentToGlobalState.get(environment); if (!globalState.globalTeardownPromise) { - if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise)) - globalState.globalTeardownPromise = this.runHook(environment, 'globalTeardown', []); + if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise)) { + globalState.globalTeardownPromise = (async () => { + let result = true; + for (const hook of environment.hooks('globalTeardown')) { + if (!await this._runHook(null /* worker */, null /* testRun */, hook, environment.name(), [])) + result = false; + } + return result; + })(); + } } if (!globalState.globalTeardownPromise) return true; @@ -398,18 +433,18 @@ class HookRunner { } async _willStartHook(worker, testRun, hook, fullName) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}"`); + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); } async _didFailHook(worker, testRun, hook, fullName, message, error) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}"`); + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); if (message) this._testRunner._result.addError(message, error, worker); this._testRunner._result.setResult(TestResult.Crashed, message); } async _didCompleteHook(worker, testRun, hook, fullName) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}"`); + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); } } diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index 3655c292ea4f5..de5765e250ae3 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -281,27 +281,34 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit it('should run all hooks in proper order', async() => { const log = []; const t = new Runner(); - const e = { - name() { return 'env'; }, - beforeAll() { log.push('env:beforeAll'); }, - afterAll() { log.push('env:afterAll'); }, - beforeEach() { log.push('env:beforeEach'); }, - afterEach() { log.push('env:afterEach'); }, - }; - const e2 = { - name() { return 'env2'; }, - beforeAll() { log.push('env2:beforeAll'); }, - afterAll() { log.push('env2:afterAll'); }, - }; + const e = new Environment('env'); + e.beforeAll(() => log.push('env:beforeAll')); + e.afterAll(() => log.push('env:afterAll')); + e.beforeEach(() => log.push('env:beforeEach')); + e.afterEach(() => log.push('env:afterEach')); + const e2 = new Environment('env2'); + e2.beforeAll(() => log.push('env2:beforeAll')); + e2.afterAll(() => log.push('env2:afterAll')); t.beforeAll(() => log.push('root:beforeAll')); - t.beforeEach(() => log.push('root:beforeEach')); + t.beforeEach(() => log.push('root:beforeEach1')); + t.beforeEach(() => log.push('root:beforeEach2')); t.it('uno', () => log.push('test #1')); t.describe('suite1', () => { - t.beforeAll(() => log.push('suite:beforeAll')); + t.beforeAll(() => log.push('suite:beforeAll1')); + t.beforeAll(() => log.push('suite:beforeAll2')); t.beforeEach(() => log.push('suite:beforeEach')); t.it('dos', () => log.push('test #2')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before1')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before2')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after1')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after2')); t.it('tres', () => log.push('test #3')); - t.afterEach(() => log.push('suite:afterEach')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before1')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before2')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after1')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after2')); + t.afterEach(() => log.push('suite:afterEach1')); + t.afterEach(() => log.push('suite:afterEach2')); t.afterAll(() => log.push('suite:afterAll')); }); t.it('cuatro', () => log.push('test #4')); @@ -319,26 +326,41 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit t.suites()[t.suites().length - 1].addEnvironment(e); t.suites()[t.suites().length - 1].addEnvironment(e2); t.afterEach(() => log.push('root:afterEach')); - t.afterAll(() => log.push('root:afterAll')); + t.afterAll(() => log.push('root:afterAll1')); + t.afterAll(() => log.push('root:afterAll2')); await t.run(); expect(log).toEqual([ 'root:beforeAll', - 'root:beforeEach', + 'root:beforeEach1', + 'root:beforeEach2', 'test #1', 'root:afterEach', - 'suite:beforeAll', + 'suite:beforeAll1', + 'suite:beforeAll2', - 'root:beforeEach', + 'root:beforeEach1', + 'root:beforeEach2', 'suite:beforeEach', + 'test:before1', + 'test:before2', 'test #2', - 'suite:afterEach', + 'test:after1', + 'test:after2', + 'suite:afterEach1', + 'suite:afterEach2', 'root:afterEach', - 'root:beforeEach', + 'root:beforeEach1', + 'root:beforeEach2', 'suite:beforeEach', + 'test:before1', + 'test:before2', 'test #3', - 'suite:afterEach', + 'test:after1', + 'test:after2', + 'suite:afterEach1', + 'suite:afterEach2', 'root:afterEach', 'suite:afterAll', @@ -346,14 +368,16 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit 'env:beforeAll', 'env2:beforeAll', - 'root:beforeEach', + 'root:beforeEach1', + 'root:beforeEach2', 'env:beforeEach', 'test #4', 'env:afterEach', 'root:afterEach', 'suite2:beforeAll', - 'root:beforeEach', + 'root:beforeEach1', + 'root:beforeEach2', 'env:beforeEach', 'test #5', 'env:afterEach', @@ -363,26 +387,23 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit 'env2:afterAll', 'env:afterAll', - 'root:afterAll', + 'root:afterAll1', + 'root:afterAll2', ]); }); it('should remove environment', async() => { const log = []; const t = new Runner(); - const e = { - name() { return 'env'; }, - beforeAll() { log.push('env:beforeAll'); }, - afterAll() { log.push('env:afterAll'); }, - beforeEach() { log.push('env:beforeEach'); }, - afterEach() { log.push('env:afterEach'); }, - }; - const e2 = { - name() { return 'env2'; }, - beforeAll() { log.push('env2:beforeAll'); }, - afterAll() { log.push('env2:afterAll'); }, - beforeEach() { log.push('env2:beforeEach'); }, - afterEach() { log.push('env2:afterEach'); }, - }; + const e = new Environment('env'); + e.beforeAll(() => log.push('env:beforeAll')); + e.afterAll(() => log.push('env:afterAll')); + e.beforeEach(() => log.push('env:beforeEach')); + e.afterEach(() => log.push('env:afterEach')); + const e2 = new Environment('env2'); + e2.beforeAll(() => log.push('env2:beforeAll')); + e2.afterAll(() => log.push('env2:afterAll')); + e2.beforeEach(() => log.push('env2:beforeEach')); + e2.afterEach(() => log.push('env2:afterEach')); t.it('uno', () => log.push('test #1')); t.tests()[0].addEnvironment(e).addEnvironment(e2).removeEnvironment(e); await t.run();