Generate into a separate directory, parallelize different builds

Write generated files into a separate directory to remove build step order hazards. This allows parallelizing the different builds since out/ is no longer the "special" one that builds before everything else.
gpuweb · Nov 2, 2023 · 02eaf7c · 02eaf7c
1 parent e92730e
commit 02eaf7c
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 42 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@
 .vscode/
 
 # Build files
+/gen/
 /out/
 /out-wpt/
 /out-node/

diff --git a/Gruntfile.js b/Gruntfile.js
@@ -4,6 +4,8 @@
 
 const timer = require('grunt-timer');
 
+const kAllSuites = ['webgpu', 'stress', 'manual', 'unittests', 'demo'];
+
 module.exports = function (grunt) {
   timer.init(grunt);
 
@@ -12,7 +14,15 @@ module.exports = function (grunt) {
     pkg: grunt.file.readJSON('package.json'),
 
     clean: {
-      out: ['out/', 'out-wpt/', 'out-node/'],
+      all: ['out/', 'out-wpt/', 'out-node/', 'gen/'],
+      'leftover-map-files-out': [
+        'out/common/internal/version.js.map',
+        'out/*/listing.js.map',
+      ],
+      'leftover-map-files-out-wpt': [
+        'out-wpt/common/internal/version.js.map',
+        'out-wpt/*/listing.js.map',
+      ],
     },
 
     run: {
@@ -21,14 +31,12 @@ module.exports = function (grunt) {
         args: ['tools/gen_version'],
       },
       'generate-listings': {
-        // Overwrites the listings.js files in out/. Must run before copy:out-wpt-generated;
-        // must not run before run:build-out (if it is run).
         cmd: 'node',
-        args: ['tools/gen_listings', 'out/', 'src/webgpu', 'src/stress', 'src/manual', 'src/unittests', 'src/demo'],
+        args: ['tools/gen_listings', 'gen/', ...kAllSuites.map(s => 'src/' + s)],
       },
       validate: {
         cmd: 'node',
-        args: ['tools/validate', 'src/webgpu', 'src/stress', 'src/manual', 'src/unittests', 'src/demo'],
+        args: ['tools/validate', ...kAllSuites.map(s => 'src/' + s)],
       },
       'validate-cache': {
         cmd: 'node',
@@ -51,7 +59,6 @@ module.exports = function (grunt) {
         args: ['tools/run_node', 'unittests:*'],
       },
       'build-out': {
-        // Must run before run:generate-listings, which will overwrite some files.
         cmd: 'node',
         args: [
           'node_modules/@babel/cli/bin/babel',
@@ -142,15 +149,17 @@ module.exports = function (grunt) {
     },
 
     copy: {
-      'out-wpt-generated': {
+      'gen-to-out': {
         files: [
           // Must run after run:generate-version and run:generate-listings.
-          { expand: true, cwd: 'out', src: 'common/internal/version.js', dest: 'out-wpt/' },
-          { expand: true, cwd: 'out', src: 'webgpu/listing.js', dest: 'out-wpt/' },
+          { expand: true, cwd: 'gen', src: 'common/internal/version.js', dest: 'out/' },
+          { expand: true, cwd: 'gen', src: '*/listing.js', dest: 'out/' },
         ],
       },
-      'out-wpt-htmlfiles': {
+      'gen-to-out-wpt': {
         files: [
+          { expand: true, cwd: 'gen', src: 'common/internal/version.js', dest: 'out-wpt/' },
+          { expand: true, cwd: 'gen', src: 'webgpu/listing.js', dest: 'out-wpt/' },
           { expand: true, cwd: 'src', src: 'webgpu/**/*.html', dest: 'out-wpt/' },
         ],
       },
@@ -166,6 +175,14 @@ module.exports = function (grunt) {
     },
 
     parallel: {
+      'build-all': {
+        options: { grunt: true },
+        tasks: [
+          'build-standalone',
+          'build-wpt',
+          'run:build-out-node',
+        ]
+      },
       'generate-wpt-cts-html-all': {
         options: { grunt: true },
         tasks: [
@@ -180,7 +197,7 @@ module.exports = function (grunt) {
           'ts:check',
         ]
       },
-      'all-checks': {
+      'all-checks-and-docs': {
         options: { grunt: true },
         tasks: [
           'run:validate',
@@ -210,47 +227,54 @@ module.exports = function (grunt) {
     helpMessageTasks.push({ name, desc });
   }
 
-  grunt.registerTask('build-standalone', 'Build out/ (no listings, no checks, no WPT)', [
+  grunt.registerTask('build-standalone', 'Build out/ (no checks; run after code generate-*)', [
     'run:build-out',
     'run:copy-assets',
-    'run:generate-version',
+    'gen-to-out',
   ]);
-  grunt.registerTask('build-wpt', 'Build out-wpt/ (no checks; run after generate-listings)', [
+  grunt.registerTask('build-wpt', 'Build out-wpt/ (no checks; run after generate-*)', [
     'run:build-out-wpt',
     'run:copy-assets-wpt',
+    'gen-to-out-wpt',
     'run:autoformat-out-wpt',
-    'run:generate-version',
-    'copy:out-wpt-generated',
-    'copy:out-wpt-htmlfiles',
     'parallel:generate-wpt-cts-html-all',
   ]);
+  grunt.registerTask('generate-all', 'Create generated files in gen/', [
+    'run:generate-version',
+    'run:generate-listings',
+  ]);
+  grunt.registerTask('gen-to-out', 'Copy generated files to out/ (run after generate-all)', [
+    'copy:gen-to-out',
+    'clean:leftover-map-files-out',
+  ]);
+  grunt.registerTask('gen-to-out-wpt', 'Copy generated files to out-wpt/ (run after generate-all)', [
+    'copy:gen-to-out-wpt',
+    'clean:leftover-map-files-out-wpt',
+  ]);
   grunt.registerTask('build-done-message', () => {
     process.stderr.write('\nBuild completed! Running checks/tests');
   });
 
   registerTaskAndAddToHelp('pre', 'Run all presubmit checks: standalone+wpt+typecheck+unittest+lint', [
     'clean',
-    'build-standalone',
-    'run:generate-listings',
-    'build-wpt',
-    'run:build-out-node',
+    'generate-all',
+    'parallel:build-all',
     'build-done-message',
-    'parallel:all-checks',
+    'parallel:all-checks-and-docs',
   ]);
   registerTaskAndAddToHelp('standalone', 'Build standalone and typecheck', [
+    'generate-all',
     'build-standalone',
-    'run:generate-listings',
     'build-done-message',
     'parallel:validate-and-typecheck',
   ]);
   registerTaskAndAddToHelp('wpt', 'Build for WPT and typecheck', [
-    'run:generate-listings',
+    'generate-all',
     'build-wpt',
     'build-done-message',
     'parallel:validate-and-typecheck',
   ]);
   registerTaskAndAddToHelp('unittest', 'Build standalone, typecheck, and unittest', [
-    'standalone',
     'run:unittest',
   ]);
   registerTaskAndAddToHelp('check', 'Just typecheck', [
@@ -263,7 +287,7 @@ module.exports = function (grunt) {
   addExistingTaskToHelp('clean', 'Clean out/ and out-wpt/');
 
   grunt.registerTask('default', '', () => {
-    console.error('\nAvailable tasks (see grunt --help for info):');
+    console.error('\nAvailable tasks:');
     for (const { name, desc } of helpMessageTasks) {
       console.error(`$ grunt ${name}`);
       console.error(`  ${desc}`);

diff --git a/src/common/tools/gen_listings.ts b/src/common/tools/gen_listings.ts
@@ -52,12 +52,5 @@ for (const suiteDir of argv.slice(3)) {
 export const listing = ${JSON.stringify(listing, undefined, 2)};
 `
     );
-
-    // If there was a sourcemap for the file we just replaced, delete it.
-    try {
-      fs.unlinkSync(outFile + '.map');
-    } catch (ex) {
-      // ignore if file didn't exist
-    }
   });
 }
diff --git a/tools/gen_version b/tools/gen_version
@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-// Get the current git hash, and save (overwrite) it into out/framework/version.js
+// Get the current git hash, and save it into gen/.../version.js
 // so it can be read when running inside the browser.
 
 /* eslint-disable no-console */
@@ -16,18 +16,13 @@ if (!fs.existsSync(myself)) {
 
 const { version } = require('../src/common/tools/version.ts');
 
-fs.mkdirSync('./out/common/internal', { recursive: true });
-// Overwrite the version.js generated by TypeScript compilation.
+fs.mkdirSync('./gen/common/internal', { recursive: true });
+// This will be copied into the various other build directories.
 fs.writeFileSync(
-  './out/common/internal/version.js',
+  './gen/common/internal/version.js',
   `\
 // AUTO-GENERATED - DO NOT EDIT. See ${myself}.
 
 export const version = '${version}';
 `
 );
-
-// Since the generated version.js was overwritten, its source map is no longer relevant.
-try {
-  fs.unlinkSync('./out/common/internal/version.js.map');
-} catch (ex) { }