minor #501 Fix various issues related to functional tests (Lyrkan)

This PR was merged into the master branch.

Discussion
----------

Fix various issues related to functional tests

This PR fixes various things that currently make the tests result indicators quite unreliable:

* Set a `DISABLE_UNSTABLE_CHECKS` environment variable and use it to disable some checks related to filenames with hashes that are subject to frequent changes (for instance `runtime.xxxxxxxx.js` when updating Webpack). This only applies to "Lowest versions of the dependencies" and "Highest versions of the dependencies" jobs.

* Fix the Vue.js + Typescript test that was silently failing in the "Lowest versions of the dependencies" job (no idea why it worked for the other ones...)

* Increase default timeouts of functional tests from 8s to 10s (since they were frequently failing because of that)

* Restore the standard output at the beginning of the compiler callback in `runWebpack()` (doing it later could end up hiding the rest of the job's output if an exception was thrown)

Commits
-------

b007c0e Fix various issues related to functional tests

Author: Ryan Weaver

.travis.yml

@@ -11,15 +11,19 @@ matrix:
     - name: 'Lowest versions of the dependencies'
       os: linux
       node_js: "10"
-      env: JOB_PART=test
+      env:
+        - JOB_PART=test
+        - DISABLE_UNSTABLE_CHECKS=1
       install:
         - rm yarn.lock
         - node ./scripts/force-lowest-dependencies
         - yarn
     - name: 'Highest versions of the dependencies'
       os: linux
       node_js: "10"
-      env: JOB_PART=test
+      env:
+        - JOB_PART=test
+        - DISABLE_UNSTABLE_CHECKS=1
       install:
         - rm yarn.lock
         - yarn

fixtures/vuejs-typescript/tsconfig.json

@@ -3,9 +3,10 @@
         "target": "es6",
         "strict": true,
         "module": "es2015",
-        "moduleResolution": "node"
+        "moduleResolution": "node",
+        "allowSyntheticDefaultImports": true
     },
     "include": [
-        "./**/*.ts",
+        "./**/*.ts"
     ]
 }

test/bin/encore.js

@@ -19,7 +19,7 @@ const exec = require('child_process').exec;
 
 describe('bin/encore.js', function() {
     // being functional tests, these can take quite long
-    this.timeout(8000);
+    this.timeout(10000);
 
     it('Basic smoke test', (done) => {
         testSetup.emptyTmpDir();

test/functional.js

@@ -62,7 +62,7 @@ function getEntrypointData(config, entryName) {
 
 describe('Functional tests using webpack', function() {
     // being functional tests, these can take quite long
-    this.timeout(8000);
+    this.timeout(10000);
 
     before(() => {
         testSetup.emptyTmpDir();
@@ -388,14 +388,16 @@ describe('Functional tests using webpack', function() {
                 config.enableVersioning(true);
 
                 testSetup.runWebpack(config, (webpackAssert) => {
-                    expect(config.outputPath).to.be.a.directory()
-                        .with.files([
-                            'main.89eb104b.js',
-                            'styles.8ec31654.css',
-                            'manifest.json',
-                            'entrypoints.json',
-                            'runtime.3d179b24.js',
-                        ]);
+                    if (!process.env.DISABLE_UNSTABLE_CHECKS) {
+                        expect(config.outputPath).to.be.a.directory()
+                            .with.files([
+                                'main.89eb104b.js',
+                                'styles.8ec31654.css',
+                                'manifest.json',
+                                'entrypoints.json',
+                                'runtime.3d179b24.js',
+                            ]);
+                    }
 
                     webpackAssert.assertOutputFileContains(
                         'styles.8ec31654.css',
@@ -489,17 +491,19 @@ describe('Functional tests using webpack', function() {
             config.enableVersioning(true);
 
             testSetup.runWebpack(config, (webpackAssert) => {
-                expect(config.outputPath).to.be.a.directory()
-                    .with.files([
-                        '0.590a68c7.js', // chunks are also versioned
-                        '0.8ec31654.css',
-                        'main.4a5effdb.js',
-                        'h1.8ec31654.css',
-                        'bg.0ec2735b.css',
-                        'manifest.json',
-                        'entrypoints.json',
-                        'runtime.b84a9b43.js',
-                    ]);
+                if (!process.env.DISABLE_UNSTABLE_CHECKS) {
+                    expect(config.outputPath).to.be.a.directory()
+                        .with.files([
+                            '0.590a68c7.js', // chunks are also versioned
+                            '0.8ec31654.css',
+                            'main.4a5effdb.js',
+                            'h1.8ec31654.css',
+                            'bg.0ec2735b.css',
+                            'manifest.json',
+                            'entrypoints.json',
+                            'runtime.b84a9b43.js',
+                        ]);
+                }
 
                 expect(path.join(config.outputPath, 'images')).to.be.a.directory()
                     .with.files([
@@ -1159,8 +1163,8 @@ module.exports = {
         });
 
         it('TypeScript is compiled and type checking is done in a separate process!', (done) => {
-            this.timeout(8000);
-            setTimeout(done, 7000);
+            this.timeout(10000);
+            setTimeout(done, 9000);
 
             const config = createWebpackConfig('www/build', 'dev');
             config.setPublicPath('/build');
@@ -1687,26 +1691,28 @@ module.exports = {
                 }]);
 
                 testSetup.runWebpack(config, (webpackAssert) => {
-                    expect(config.outputPath).to.be.a.directory()
-                        .with.files([
-                            'entrypoints.json',
-                            'runtime.6cf710cd.js',
-                            'main.d85842cc.js',
-                            'manifest.json',
-                            'symfony_logo.ea1ca6f7.png',
-                            'symfony_logo_alt.f27119c2.png',
-                        ]);
+                    if (!process.env.DISABLE_UNSTABLE_CHECKS) {
+                        expect(config.outputPath).to.be.a.directory()
+                            .with.files([
+                                'entrypoints.json',
+                                'runtime.6cf710cd.js',
+                                'main.d85842cc.js',
+                                'manifest.json',
+                                'symfony_logo.ea1ca6f7.png',
+                                'symfony_logo_alt.f27119c2.png',
+                            ]);
+
+                        webpackAssert.assertManifestPath(
+                            'build/main.js',
+                            '/build/main.d85842cc.js'
+                        );
+                    }
 
                     expect(path.join(config.outputPath, 'assets')).to.be.a.directory()
                         .with.files([
                             'Roboto.woff2',
                         ]);
 
-                    webpackAssert.assertManifestPath(
-                        'build/main.js',
-                        '/build/main.d85842cc.js'
-                    );
-
                     webpackAssert.assertManifestPath(
                         'build/symfony_logo.png',
                         '/build/symfony_logo.ea1ca6f7.png'

test/helpers/setup.js

@@ -73,6 +73,8 @@ function runWebpack(webpackConfig, callback, allowCompilationError = false) {
 
         const compiler = webpack(configGenerator(webpackConfig));
         compiler.run((err, stats) => {
+            // Restore stdout
+            process.stdout.write = stdoutWrite;
 
             if (err) {
                 console.error(err.stack || err);
@@ -95,8 +97,6 @@ function runWebpack(webpackConfig, callback, allowCompilationError = false) {
                 console.warn(info.warnings);
             }
 
-            // Restore stdout and then call the callback
-            process.stdout.write = stdoutWrite;
             callback(assertUtil(webpackConfig), stats, stdOutContents.join('\n'));
         });
     } catch (e) {