Making tests more resilient to failure by fuzzy checking some hash filenames

Author: weaverryan

test/functional.js

@@ -418,16 +418,13 @@ describe('Functional tests using webpack', function() {
                 config.enableVersioning(true);
 
                 testSetup.runWebpack(config, (webpackAssert) => {
-                    if (!process.env.DISABLE_UNSTABLE_CHECKS) {
-                        expect(config.outputPath).to.be.a.directory()
-                            .with.files([
-                                'main.8b8ca4a9.js',
-                                'styles.79943add.css',
-                                'manifest.json',
-                                'entrypoints.json',
-                                'runtime.49688a3c.js',
-                            ]);
-                    }
+                    webpackAssert.assertDirectoryContents([
+                        'main.[hash:8].js',
+                        'styles.[hash:8].css',
+                        'manifest.json',
+                        'entrypoints.json',
+                        'runtime.[hash:8].js',
+                    ]);
 
                     webpackAssert.assertOutputFileContains(
                         'styles.79943add.css',
@@ -521,19 +518,16 @@ describe('Functional tests using webpack', function() {
             config.enableVersioning(true);
 
             testSetup.runWebpack(config, (webpackAssert) => {
-                if (!process.env.DISABLE_UNSTABLE_CHECKS) {
-                    expect(config.outputPath).to.be.a.directory()
-                        .with.files([
-                            'js_no_require_js-css_h1_style_css.456c237a.js', // chunks are also versioned
-                            'js_no_require_js-css_h1_style_css.79943add.css',
-                            'main.04316e30.js',
-                            'h1.79943add.css',
-                            'bg.2eff0999.css',
-                            'manifest.json',
-                            'entrypoints.json',
-                            'runtime.17e58ffe.js',
-                        ]);
-                }
+                webpackAssert.assertDirectoryContents([
+                    'js_no_require_js-css_h1_style_css.[hash:8].js', // chunks are also versioned
+                    'js_no_require_js-css_h1_style_css.[hash:8].css',
+                    'main.[hash:8].js',
+                    'h1.[hash:8].css',
+                    'bg.[hash:8].css',
+                    'manifest.json',
+                    'entrypoints.json',
+                    'runtime.[hash:8].js',
+                ]);
 
                 expect(path.join(config.outputPath, 'images')).to.be.a.directory()
                     .with.files([
@@ -2167,22 +2161,19 @@ module.exports = {
                 }]);
 
                 testSetup.runWebpack(config, (webpackAssert) => {
-                    if (!process.env.DISABLE_UNSTABLE_CHECKS) {
-                        expect(config.outputPath).to.be.a.directory()
-                            .with.files([
-                                'entrypoints.json',
-                                'runtime.fbc90386.js',
-                                'main.06a6c20f.js',
-                                'manifest.json',
-                                'symfony_logo.91beba37.png',
-                                'symfony_logo_alt.f880ba14.png',
-                            ]);
-
-                        webpackAssert.assertManifestPath(
-                            'build/main.js',
-                            '/build/main.06a6c20f.js'
-                        );
-                    }
+                    webpackAssert.assertDirectoryContents([
+                        'entrypoints.json',
+                        'runtime.fbc90386.js',
+                        'main.06a6c20f.js',
+                        'manifest.json',
+                        'symfony_logo.91beba37.png',
+                        'symfony_logo_alt.f880ba14.png',
+                    ]);
+
+                    webpackAssert.assertManifestPath(
+                        'build/main.js',
+                        '/build/main.[hash:8].js'
+                    );
 
                     expect(path.join(config.outputPath, 'assets')).to.be.a.directory()
                         .with.files([

test/helpers/assert.js

@@ -13,6 +13,7 @@ const path = require('path');
 const fs = require('fs');
 const chai = require('chai');
 const expect = chai.expect;
+const regexEscaper = require('../../lib/utils/regexp-escaper');
 
 const loadManifest = function(webpackConfig) {
     return JSON.parse(
@@ -30,6 +31,27 @@ const readOutputFile = function(webpackConfig, filePath) {
     return fs.readFileSync(fullPath, 'utf8');
 };
 
+/**
+ * Returns a regex to use to match this filename
+ *
+ * @param {string} filename Filename with possible [hash:8] wildcard
+ * @return {RegExp}
+ */
+const convertFilenameToMatcher = function(filename) {
+    const hashMatch = filename.match(/\[hash:(\d+)\]/);
+
+    if (hashMatch === null) {
+        return new RegExp(regexEscaper(filename));
+    }
+
+    const [hashString, hashLength] = hashMatch;
+
+    return new RegExp(
+        regexEscaper(filename)
+            .replace(regexEscaper(hashString), `([a-z0-9_-]){${hashLength}}`)
+    );
+};
+
 class Assert {
     /**
      * @param {WebpackConfig} webpackConfig
@@ -107,8 +129,10 @@ class Assert {
 
         this.assertManifestKeyExists(sourcePath);
 
-        if (manifestData[sourcePath] !== expectedDestinationPath) {
-            throw new Error(`source path ${sourcePath} expected to be set to ${expectedDestinationPath}, was actually ${manifestData[sourcePath]}`);
+        const expectedRegex = convertFilenameToMatcher(expectedDestinationPath);
+
+        if (!manifestData[sourcePath].match(expectedRegex)) {
+            throw new Error(`source path ${sourcePath} expected to match pattern ${expectedDestinationPath}, was actually ${manifestData[sourcePath]}`);
         }
     }
 
@@ -168,6 +192,57 @@ class Assert {
 
         expect(JSON.stringify(actualData, null, 2)).to.equal(JSON.stringify(expectedData, null, 2));
     }
+
+    /**
+     * Verifies that the directory contains the array of files.
+     *
+     * The expectedFiles can contain a [hash:8] syntax in case
+     * the file is versioned - e.g. main.[hash:8].js, which would
+     * match a real file like main.abcd1234.js.
+     *
+     * @param {Array} expectedFiles
+     * @param {string} directory relative to output to check
+     * @returns {void}
+     */
+    assertDirectoryContents(expectedFiles, directory = '') {
+        const targetDirectory = path.join(this.webpackConfig.outputPath, directory);
+
+        expect(targetDirectory).to.be.a.directory();
+
+        const expectedFileStrings = {};
+        expectedFiles.forEach((expectedFile) => {
+            expectedFileStrings[expectedFile] = convertFilenameToMatcher(expectedFile);
+        });
+
+        const actualFiles = fs.readdirSync(targetDirectory);
+        actualFiles.forEach((foundFile) => {
+            // filter out directories
+            if (fs.statSync(path.join(targetDirectory, foundFile)).isDirectory()) {
+                return;
+            }
+
+            let matchIsFound = false;
+
+            for (const originalFilename of Object.keys(expectedFileStrings)) {
+                const filenameRegex = expectedFileStrings[originalFilename];
+
+                if (foundFile.match(filenameRegex)) {
+                    matchIsFound = true;
+                    delete expectedFileStrings[originalFilename];
+
+                    break;
+                }
+            }
+
+            if (!matchIsFound) {
+                throw new Error(`File "${foundFile}" was found in directory but was not expected. Expected patterns where ${expectedFiles.join(', ')}`);
+            }
+        });
+
+        if (Object.keys(expectedFileStrings).length > 0) {
+            throw new Error(`Files ${Object.keys(expectedFileStrings).join(', ')} were expected to be found in the directory but were not. Actual files: ${actualFiles.join(', ')}`);
+        }
+    }
 }
 
 module.exports = function(webpackConfig) {