diff --git a/bin/packages/build-worker.js b/bin/packages/build-worker.js new file mode 100644 index 00000000000000..7e3e636c013a1d --- /dev/null +++ b/bin/packages/build-worker.js @@ -0,0 +1,159 @@ +/** + * External dependencies + */ +const { promisify } = require( 'util' ); +const fs = require( 'fs' ); +const path = require( 'path' ); +const babel = require( '@babel/core' ); +const makeDir = require( 'make-dir' ); +const sass = require( 'node-sass' ); +const postcss = require( 'postcss' ); + +/** + * Internal dependencies + */ +const getBabelConfig = require( './get-babel-config' ); + +/** + * Path to packages directory. + * + * @type {string} + */ +const PACKAGES_DIR = path.resolve( __dirname, '../../packages' ); + +/** + * Mapping of JavaScript environments to corresponding build output. + * + * @type {Object} + */ +const JS_ENVIRONMENTS = { + main: 'build', + module: 'build-module', +}; + +/** + * Promisified fs.readFile. + * + * @type {Function} + */ +const readFile = promisify( fs.readFile ); + +/** + * Promisified fs.writeFile. + * + * @type {Function} + */ +const writeFile = promisify( fs.writeFile ); + +/** + * Promisified sass.render. + * + * @type {Function} + */ +const renderSass = promisify( sass.render ); + +/** + * Get the package name for a specified file + * + * @param {string} file File name + * @return {string} Package name + */ +function getPackageName( file ) { + return path.relative( PACKAGES_DIR, file ).split( path.sep )[ 0 ]; +} + +/** + * Get Build Path for a specified file. + * + * @param {string} file File to build + * @param {string} buildFolder Output folder + * @return {string} Build path + */ +function getBuildPath( file, buildFolder ) { + const pkgName = getPackageName( file ); + const pkgSrcPath = path.resolve( PACKAGES_DIR, pkgName, 'src' ); + const pkgBuildPath = path.resolve( PACKAGES_DIR, pkgName, buildFolder ); + const relativeToSrcPath = path.relative( pkgSrcPath, file ); + return path.resolve( pkgBuildPath, relativeToSrcPath ); +} + +/** + * Object of build tasks per file extension. + * + * @type {Object} + */ +const BUILD_TASK_BY_EXTENSION = { + async '.scss'( file ) { + const outputFile = getBuildPath( file.replace( '.scss', '.css' ), 'build-style' ); + const outputFileRTL = getBuildPath( file.replace( '.scss', '-rtl.css' ), 'build-style' ); + + const [ , contents ] = await Promise.all( [ + makeDir( path.dirname( outputFile ) ), + readFile( file, 'utf8' ), + ] ); + + const builtSass = await renderSass( { + file, + includePaths: [ path.resolve( __dirname, '../../assets/stylesheets' ) ], + data: ( + [ + 'colors', + 'breakpoints', + 'variables', + 'mixins', + 'animations', + 'z-index', + ].map( ( imported ) => `@import "${ imported }";` ).join( ' ' ) + + contents + ), + } ); + + const result = await postcss( require( './post-css-config' ) ).process( builtSass.css, { + from: 'src/app.css', + to: 'dest/app.css', + } ); + + const resultRTL = await postcss( [ require( 'rtlcss' )() ] ).process( result.css, { + from: 'src/app.css', + to: 'dest/app.css', + } ); + + await Promise.all( [ + writeFile( outputFile, result.css ), + writeFile( outputFileRTL, resultRTL.css ), + ] ); + }, + + async '.js'( file ) { + for ( const [ environment, buildDir ] of Object.entries( JS_ENVIRONMENTS ) ) { + const destPath = getBuildPath( file, buildDir ); + const babelOptions = getBabelConfig( environment, file.replace( PACKAGES_DIR, '@wordpress' ) ); + + const [ , transformed ] = await Promise.all( [ + makeDir( path.dirname( destPath ) ), + babel.transformFileAsync( file, babelOptions ), + ] ); + + await Promise.all( [ + writeFile( destPath + '.map', JSON.stringify( transformed.map ) ), + writeFile( destPath, transformed.code + '\n//# sourceMappingURL=' + path.basename( destPath ) + '.map' ), + ] ); + } + }, +}; + +module.exports = async ( file, callback ) => { + const extension = path.extname( file ); + const task = BUILD_TASK_BY_EXTENSION[ extension ]; + + if ( ! task ) { + return; + } + + try { + await task( file ); + callback(); + } catch ( error ) { + callback( error ); + } +}; diff --git a/bin/packages/build.js b/bin/packages/build.js index bb6954b4102e7c..0f77d13ff06df3 100755 --- a/bin/packages/build.js +++ b/bin/packages/build.js @@ -1,220 +1,91 @@ -/** - * script to build WordPress packages into `build/` directory. - * - * Example: - * node ./scripts/build.js - */ +/* eslint-disable no-console */ /** * External dependencies */ -const fs = require( 'fs' ); const path = require( 'path' ); -const glob = require( 'glob' ); -const babel = require( '@babel/core' ); -const chalk = require( 'chalk' ); -const mkdirp = require( 'mkdirp' ); -const sass = require( 'node-sass' ); -const postcss = require( 'postcss' ); -const deasync = require( 'deasync' ); - -/** - * Internal dependencies - */ -const getPackages = require( './get-packages' ); -const getBabelConfig = require( './get-babel-config' ); - -/** - * Module Constants - */ -const PACKAGES_DIR = path.resolve( __dirname, '../../packages' ); -const SRC_DIR = 'src'; -const BUILD_DIR = { - main: 'build', - module: 'build-module', - style: 'build-style', -}; -const DONE = chalk.reset.inverse.bold.green( ' DONE ' ); - -/** - * Get the package name for a specified file - * - * @param {string} file File name - * @return {string} Package name - */ -function getPackageName( file ) { - return path.relative( PACKAGES_DIR, file ).split( path.sep )[ 0 ]; -} - -const isJsFile = ( filepath ) => { - return /.\.js$/.test( filepath ); -}; +const glob = require( 'fast-glob' ); +const ProgressBar = require( 'progress' ); +const workerFarm = require( 'worker-farm' ); +const { Readable } = require( 'stream' ); -const isScssFile = ( filepath ) => { - return /.\.scss$/.test( filepath ); -}; +const files = process.argv.slice( 2 ); /** - * Get Build Path for a specified file + * Path to packages directory. * - * @param {string} file File to build - * @param {string} buildFolder Output folder - * @return {string} Build path + * @type {string} */ -function getBuildPath( file, buildFolder ) { - const pkgName = getPackageName( file ); - const pkgSrcPath = path.resolve( PACKAGES_DIR, pkgName, SRC_DIR ); - const pkgBuildPath = path.resolve( PACKAGES_DIR, pkgName, buildFolder ); - const relativeToSrcPath = path.relative( pkgSrcPath, file ); - return path.resolve( pkgBuildPath, relativeToSrcPath ); -} +const PACKAGES_DIR = path.resolve( __dirname, '../../packages' ); -/** - * Given a list of scss and js filepaths, divide them into sets them and rebuild. - * - * @param {Array} files list of files to rebuild - */ -function buildFiles( files ) { - // Reduce files into a unique sets of javaScript files and scss packages. - const buildPaths = files.reduce( ( accumulator, filePath ) => { - if ( isJsFile( filePath ) ) { - accumulator.jsFiles.add( filePath ); - } else if ( isScssFile( filePath ) ) { - const pkgName = getPackageName( filePath ); - const pkgPath = path.resolve( PACKAGES_DIR, pkgName ); - accumulator.scssPackagePaths.add( pkgPath ); - } - return accumulator; - }, { jsFiles: new Set(), scssPackagePaths: new Set() } ); +let onFileComplete = () => {}; - buildPaths.jsFiles.forEach( buildJsFile ); - buildPaths.scssPackagePaths.forEach( buildPackageScss ); -} +let stream; -/** - * Build a javaScript file for the required environments (node and ES5) - * - * @param {string} file File path to build - * @param {boolean} silent Show logs - */ -function buildJsFile( file, silent ) { - buildJsFileFor( file, silent, 'main' ); - buildJsFileFor( file, silent, 'module' ); -} - -/** - * Build a package's scss styles - * - * @param {string} packagePath The path to the package. - */ -function buildPackageScss( packagePath ) { - const srcDir = path.resolve( packagePath, SRC_DIR ); - const scssFiles = glob.sync( `${ srcDir }/*.scss` ); - - // Build scss files individually. - scssFiles.forEach( buildScssFile ); -} - -function buildScssFile( styleFile ) { - const outputFile = getBuildPath( styleFile.replace( '.scss', '.css' ), BUILD_DIR.style ); - const outputFileRTL = getBuildPath( styleFile.replace( '.scss', '-rtl.css' ), BUILD_DIR.style ); - mkdirp.sync( path.dirname( outputFile ) ); - const builtSass = sass.renderSync( { - file: styleFile, - includePaths: [ path.resolve( __dirname, '../../assets/stylesheets' ) ], - data: ( - [ - 'colors', - 'breakpoints', - 'variables', - 'mixins', - 'animations', - 'z-index', - ].map( ( imported ) => `@import "${ imported }";` ).join( ' ' ) + - fs.readFileSync( styleFile, 'utf8' ) - ), +if ( files.length ) { + stream = new Readable( { encoding: 'utf8' } ); + files.forEach( ( file ) => stream.push( file ) ); + stream.push( null ); +} else { + const bar = new ProgressBar( 'Build Progress: [:bar] :percent', { + width: 30, + incomplete: ' ', + total: 1, } ); - const postCSSSync = ( callback ) => { - postcss( require( './post-css-config' ) ) - .process( builtSass.css, { from: 'src/app.css', to: 'dest/app.css' } ) - .then( ( result ) => callback( null, result ) ); - }; - - const postCSSRTLSync = ( ltrCSS, callback ) => { - postcss( [ require( 'rtlcss' )() ] ) - .process( ltrCSS, { from: 'src/app.css', to: 'dest/app.css' } ) - .then( ( result ) => callback( null, result ) ); - }; - - const result = deasync( postCSSSync )(); - fs.writeFileSync( outputFile, result.css ); - - const resultRTL = deasync( postCSSRTLSync )( result ); - fs.writeFileSync( outputFileRTL, resultRTL ); -} - -/** - * Build a file for a specific environment - * - * @param {string} file File path to build - * @param {boolean} silent Show logs - * @param {string} environment Dist environment (node or es5) - */ -function buildJsFileFor( file, silent, environment ) { - const buildDir = BUILD_DIR[ environment ]; - const destPath = getBuildPath( file, buildDir ); - const babelOptions = getBabelConfig( environment, file.replace( PACKAGES_DIR, '@wordpress' ) ); - - mkdirp.sync( path.dirname( destPath ) ); - const transformed = babel.transformFileSync( file, babelOptions ); - fs.writeFileSync( destPath + '.map', JSON.stringify( transformed.map ) ); - fs.writeFileSync( destPath, transformed.code + '\n//# sourceMappingURL=' + path.basename( destPath ) + '.map' ); - - if ( ! silent ) { - process.stdout.write( - chalk.green( ' \u2022 ' ) + - path.relative( PACKAGES_DIR, file ) + - chalk.green( ' \u21D2 ' ) + - path.relative( PACKAGES_DIR, destPath ) + - '\n' - ); - } -} + bar.tick( 0 ); -/** - * Build the provided package path - * - * @param {string} packagePath absolute package path - */ -function buildPackage( packagePath ) { - const srcDir = path.resolve( packagePath, SRC_DIR ); - const jsFiles = glob.sync( `${ srcDir }/**/*.js`, { + stream = glob.stream( [ + `${ PACKAGES_DIR }/*/src/**/*.js`, + `${ PACKAGES_DIR }/*/src/*.scss`, + ], { ignore: [ - `${ srcDir }/**/test/**/*.js`, - `${ srcDir }/**/__mocks__/**/*.js`, + `**/test/**`, + `**/__mocks__/**`, ], - nodir: true, + onlyFiles: true, } ); - process.stdout.write( `${ path.basename( packagePath ) }\n` ); + // Pause to avoid data flow which would begin on the `data` event binding, + // but should wait until worker processing below. + // + // See: https://nodejs.org/api/stream.html#stream_two_reading_modes + stream + .pause() + .on( 'data', ( file ) => { + bar.total = files.push( file ); + } ); + + onFileComplete = () => { + bar.tick(); + }; +} - // Build js files individually. - jsFiles.forEach( ( file ) => buildJsFile( file, true ) ); +const worker = workerFarm( require.resolve( './build-worker' ) ); - // Build package CSS files - buildPackageScss( packagePath ); +let ended = false, + complete = 0; - process.stdout.write( `${ DONE }\n` ); -} +stream + .on( 'data', ( file ) => worker( file, ( error ) => { + onFileComplete(); -const files = process.argv.slice( 2 ); + if ( error ) { + // If an error occurs, the process can't be ended immediately since + // other workers are likely pending. Optimally, it would end at the + // earliest opportunity (after the current round of workers has had + // the chance to complete), but this is not made directly possible + // through `worker-farm`. Instead, ensure at least that when the + // process does exit, it exits with a non-zero code to reflect the + // fact that an error had occurred. + process.exitCode = 1; -if ( files.length ) { - buildFiles( files ); -} else { - process.stdout.write( chalk.inverse( '>> Building packages \n' ) ); - getPackages() - .forEach( buildPackage ); - process.stdout.write( '\n' ); -} + console.error( error ); + } + + if ( ended && ++complete === files.length ) { + workerFarm.end( worker ); + } + } ) ) + .on( 'end', () => ended = true ) + .resume(); diff --git a/package-lock.json b/package-lock.json index 0ad23808055801..00d2e2b1810a07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9352,9 +9352,9 @@ "dev": true }, "fast-glob": { - "version": "2.2.6", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.6.tgz", - "integrity": "sha512-0BvMaZc1k9F+MeWWMe8pL6YltFzZYcJsYU7D4JyDA6PAczaXvxqQQ/z+mDF7/4Mw01DeUc+i3CTKajnkANkV4w==", + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.7.tgz", + "integrity": "sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==", "dev": true, "requires": { "@mrmlnc/readdir-enhanced": "^2.2.1", @@ -9520,6 +9520,23 @@ "commondir": "^1.0.1", "make-dir": "^1.0.0", "pkg-dir": "^2.0.0" + }, + "dependencies": { + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } } }, "find-file-up": { @@ -12129,6 +12146,21 @@ "dev": true } } + }, + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true } } }, @@ -12181,6 +12213,21 @@ "supports-color": "^6.0.0" }, "dependencies": { + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + }, "supports-color": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", @@ -12214,12 +12261,27 @@ "ms": "^2.1.1" } }, + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", "dev": true }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -14239,18 +14301,18 @@ } }, "make-dir": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", - "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.0.tgz", + "integrity": "sha512-grNJDhb8b1Jm1qeqW5R/O63wUo4UXo2v2HMic6YT9i/HBlF93S8jkMgH7yugvY9ABDShH4VZMn8I+U8+fCNegw==", "dev": true, "requires": { - "pify": "^3.0.0" + "semver": "^6.0.0" }, "dependencies": { - "pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "semver": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.0.0.tgz", + "integrity": "sha512-0UewU+9rFapKFnlbirLi3byoOuhrSsli/z/ihNnvM24vgF+8sNBiI1LZPBSH9wJKUwaUbw+s3hToDLCXkrghrQ==", "dev": true } } @@ -18023,9 +18085,9 @@ "dev": true }, "progress": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz", - "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, "promise": { @@ -21164,6 +21226,15 @@ "uuid": "^3.0.1" }, "dependencies": { + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", @@ -22821,9 +22892,9 @@ "dev": true }, "worker-farm": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.6.0.tgz", - "integrity": "sha512-6w+3tHbM87WnSWnENBUvA2pxJPLhQUg5LKwUQHq3r+XPhIM+Gh2R5ycbwPCyuGbNg+lPgdcnQUhuC02kJCvffQ==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", + "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", "dev": true, "requires": { "errno": "~0.1.7" @@ -22902,6 +22973,15 @@ "write-file-atomic": "^2.0.0" }, "dependencies": { + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", diff --git a/package.json b/package.json index 527d358b982b37..1270e1adde7bd3 100644 --- a/package.json +++ b/package.json @@ -90,12 +90,12 @@ "core-js": "3.0.1", "cross-env": "3.2.4", "cssnano": "4.1.10", - "deasync": "0.1.14", "deep-freeze": "0.0.1", "doctrine": "2.1.0", "enzyme": "3.9.0", "eslint-plugin-jest": "21.5.0", "espree": "4.0.0", + "fast-glob": "2.2.7", "fbjs": "0.8.17", "fs-extra": "8.0.1", "glob": "7.1.2", @@ -106,6 +106,7 @@ "lerna": "3.14.1", "lint-staged": "8.1.5", "lodash": "4.17.11", + "make-dir": "3.0.0", "mkdirp": "0.5.1", "node-sass": "4.12.0", "node-watch": "0.6.0", @@ -113,6 +114,7 @@ "pegjs": "0.10.0", "phpegjs": "1.0.0-beta7", "postcss": "7.0.13", + "progress": "2.0.3", "react": "16.8.4", "react-dom": "16.8.4", "react-test-renderer": "16.8.4", @@ -128,7 +130,8 @@ "sprintf-js": "1.1.1", "stylelint-config-wordpress": "13.1.0", "uuid": "3.3.2", - "webpack": "4.8.3" + "webpack": "4.8.3", + "worker-farm": "1.7.0" }, "npmPackageJsonLintConfig": { "extends": "@wordpress/npm-package-json-lint-config",