Add case cache to repo

* Add the cache files to the CTS repo under `src/resources/cache`. * Compress & decompress the cache with gzip using the Compression Streams API. All the cache files sum to just over 4 meg. * Add a `src/resources/cache/hashes.json` to track the hash of the source files used to generate each of the cache files. Only rebuild the cases that are stale. * Update `gen_cache --validate` to check that all the cache files are present and up to date. Also error if there are old cache files that are no longer needed. This is run by `npm test`, which is also called by the GitHub presubmits. * Add `wpt lint` exemption for `*.bin` files Optimization: Add a cache hash file This file contains a hash of the source files used to build the cache. Only source files that have been changed need to be rebuilt. The validation step doesn't have to perform the expensive build step to know whether files are stale.
gpuweb · Nov 16, 2023 · ec19459 · ec19459
1 parent c47509f
commit ec19459
Show file tree

Hide file tree

Showing 117 changed files with 568 additions and 140 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -18,13 +18,17 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-node@v3
         with:
-          node-version: "16.x"
+          node-version: '16.x'
       - run: npm ci
+      - name: validating cache
+        run: npx grunt run:validate-cache
       - run: npm test
       - name: copy out-wpt to wpt tree
         run: |
           git clone --depth 2 https://github.com/web-platform-tests/wpt.git
           rsync -av  out-wpt/ wpt/webgpu
+      - name: adding wpt lint ignore rule for *.bin
+        run: 'echo "TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.bin" >> wpt/lint.ignore'
       - name: test wpt lint
         run: ./wpt lint
         working-directory: ./wpt
diff --git a/Gruntfile.js b/Gruntfile.js
@@ -30,9 +30,14 @@ module.exports = function (grunt) {
         cmd: 'node',
         args: ['tools/validate', ...kAllSuites.map(s => 'src/' + s)],
       },
+      'generate-cache': {
+        // Note this generates files into the src/ directory (not the gen/ directory).
+        cmd: 'node',
+        args: ['tools/gen_cache', 'src/webgpu'],
+      },
       'validate-cache': {
         cmd: 'node',
-        args: ['tools/gen_cache', 'out', 'src/webgpu', '--validate'],
+        args: ['tools/gen_cache', 'src/webgpu', '--validate'],
       },
       'write-out-wpt-cts-html': {
         // Note this generates directly into the out-wpt/ directory rather than the gen/ directory.
@@ -44,11 +49,6 @@ module.exports = function (grunt) {
         cmd: 'node',
         args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_chunked2sec.json'],
       },
-      'generate-cache': {
-        // Note this generates directly into the out/ directory rather than the gen/ directory.
-        cmd: 'node',
-        args: ['tools/gen_cache', 'out', 'src/webgpu'],
-      },
       unittest: {
         cmd: 'node',
         args: ['tools/run_node', 'unittests:*'],
@@ -115,6 +115,15 @@ module.exports = function (grunt) {
           '--copy-files'
         ],
       },
+      'copy-assets-node': {
+        cmd: 'node',
+        args: [
+          'node_modules/@babel/cli/bin/babel',
+          'src/resources/',
+          '--out-dir=out-node/resources/',
+          '--copy-files'
+        ],
+      },
       lint: {
         cmd: 'node',
         args: ['node_modules/eslint/bin/eslint', 'src/**/*.ts', '--max-warnings=0'],
@@ -223,9 +232,10 @@ module.exports = function (grunt) {
     helpMessageTasks.push({ name, desc });
   }
 
-  grunt.registerTask('generate-common', 'Generate files into gen/', [
+  grunt.registerTask('generate-common', 'Generate files into gen/ and src/', [
     'run:generate-version',
     'run:generate-listings',
+    'run:generate-cache',
   ]);
   grunt.registerTask('build-standalone', 'Build out/ (no checks; run after generate-common)', [
     'run:build-out',
@@ -240,6 +250,10 @@ module.exports = function (grunt) {
     'concurrent:write-out-wpt-cts-html-all',
     'run:autoformat-out-wpt',
   ]);
+  grunt.registerTask('build-node', 'Build out-node/ (no checks; run after generate-common)', [
+    'run:build-out-node',
+    'run:copy-assets-node',
+  ]);
   grunt.registerTask('build-all', 'Build out*/ (no checks; run after generate-common)', [
     'concurrent:all-builds',
     'build-done-message',
@@ -268,6 +282,11 @@ module.exports = function (grunt) {
     'build-wpt',
     'build-done-message',
   ]);
+  registerTaskAndAddToHelp('node', 'Build node (out-node/) (no checks)', [
+    'generate-common',
+    'build-node',
+    'build-done-message',
+  ]);
   registerTaskAndAddToHelp('checks', 'Run all checks (and build tsdoc)', [
     'concurrent:all-checks',
   ]);

diff --git a/src/common/framework/data_cache.ts b/src/common/framework/data_cache.ts
@@ -187,11 +187,11 @@ export interface Cacheable<Data> {
   /**
    * serialize() encodes `data` to a binary representation so that it can be stored in a cache file.
    */
-  serialize(data: Data): Uint8Array;
+  serialize(data: Data): Promise<Uint8Array>;
 
   /**
    * deserialize() is the inverse of serialize(), decoding the binary representation back to a Data
    * object.
    */
-  deserialize(binary: Uint8Array): Data;
+  deserialize(binary: Uint8Array): Promise<Data>;
 }
diff --git a/src/common/runtime/cmdline.ts b/src/common/runtime/cmdline.ts
@@ -3,6 +3,7 @@
 import * as fs from 'fs';
 
 import { dataCache } from '../framework/data_cache.js';
+import { getResourcePath, setBaseResourcePath } from '../framework/resources.js';
 import { globalTestConfig } from '../framework/test_config.js';
 import { DefaultTestFileLoader } from '../internal/file_loader.js';
 import { prettyPrintLog } from '../internal/logging/log_message.js';
@@ -37,6 +38,12 @@ Options:
   return sys.exit(rc);
 }
 
+if (!sys.existsSync('src/common/runtime/cmdline.ts')) {
+  console.log('Must be run from repository root');
+  usage(1);
+}
+setBaseResourcePath('out-node/resources');
+
 // The interface that exposes creation of the GPU, and optional interface to code coverage.
 interface GPUProviderModule {
   // @returns a GPU with the given flags
@@ -65,7 +72,6 @@ let printJSON = false;
 let quiet = false;
 let loadWebGPUExpectations: Promise<unknown> | undefined = undefined;
 let gpuProviderModule: GPUProviderModule | undefined = undefined;
-let dataPath: string | undefined = undefined;
 
 const queries: string[] = [];
 const gpuProviderFlags: string[] = [];
@@ -84,8 +90,6 @@ for (let i = 0; i < sys.args.length; ++i) {
       listMode = 'unimplemented';
     } else if (a === '--debug') {
       debug = true;
-    } else if (a === '--data') {
-      dataPath = sys.args[++i];
     } else if (a === '--print-json') {
       printJSON = true;
     } else if (a === '--expectations') {
@@ -132,21 +136,20 @@ Did you remember to build with code coverage instrumentation enabled?`
   }
 }
 
-if (dataPath !== undefined) {
-  dataCache.setStore({
-    load: (path: string) => {
-      return new Promise<Uint8Array>((resolve, reject) => {
-        fs.readFile(`${dataPath}/${path}`, (err, data) => {
-          if (err !== null) {
-            reject(err.message);
-          } else {
-            resolve(data);
-          }
-        });
+dataCache.setStore({
+  load: (path: string) => {
+    return new Promise<Uint8Array>((resolve, reject) => {
+      fs.readFile(getResourcePath(`cache/${path}`), (err, data) => {
+        if (err !== null) {
+          reject(err.message);
+        } else {
+          resolve(data);
+        }
       });
-    },
-  });
-}
+    });
+  },
+});
+
 if (verbose) {
   dataCache.setDebugLogger(console.log);
 }

diff --git a/src/common/runtime/server.ts b/src/common/runtime/server.ts
@@ -5,6 +5,7 @@ import * as http from 'http';
 import { AddressInfo } from 'net';
 
 import { dataCache } from '../framework/data_cache.js';
+import { getResourcePath, setBaseResourcePath } from '../framework/resources.js';
 import { globalTestConfig } from '../framework/test_config.js';
 import { DefaultTestFileLoader } from '../internal/file_loader.js';
 import { prettyPrintLog } from '../internal/logging/log_message.js';
@@ -25,7 +26,6 @@ Options:
   --colors                  Enable ANSI colors in output.
   --compat                  Run tests in compatibility mode.
   --coverage                Add coverage data to each result.
-  --data                    Path to the data cache directory.
   --verbose                 Print result/log of every test as it runs.
   --gpu-provider            Path to node module that provides the GPU implementation.
   --gpu-provider-flag       Flag to set on the gpu-provider as <flag>=<value>
@@ -71,13 +71,13 @@ if (!sys.existsSync('src/common/runtime/cmdline.ts')) {
   console.log('Must be run from repository root');
   usage(1);
 }
+setBaseResourcePath('out-node/resources');
 
 Colors.enabled = false;
 
 let emitCoverage = false;
 let verbose = false;
 let gpuProviderModule: GPUProviderModule | undefined = undefined;
-let dataPath: string | undefined = undefined;
 
 const gpuProviderFlags: string[] = [];
 for (let i = 0; i < sys.args.length; ++i) {
@@ -89,8 +89,6 @@ for (let i = 0; i < sys.args.length; ++i) {
       globalTestConfig.compatibility = true;
     } else if (a === '--coverage') {
       emitCoverage = true;
-    } else if (a === '--data') {
-      dataPath = sys.args[++i];
     } else if (a === '--gpu-provider') {
       const modulePath = sys.args[++i];
       gpuProviderModule = require(modulePath);
@@ -130,21 +128,20 @@ Did you remember to build with code coverage instrumentation enabled?`
   }
 }
 
-if (dataPath !== undefined) {
-  dataCache.setStore({
-    load: (path: string) => {
-      return new Promise<Uint8Array>((resolve, reject) => {
-        fs.readFile(`${dataPath}/${path}`, (err, data) => {
-          if (err !== null) {
-            reject(err.message);
-          } else {
-            resolve(data);
-          }
-        });
+dataCache.setStore({
+  load: (path: string) => {
+    return new Promise<Uint8Array>((resolve, reject) => {
+      fs.readFile(getResourcePath(`cache/${path}`), (err, data) => {
+        if (err !== null) {
+          reject(err.message);
+        } else {
+          resolve(data);
+        }
       });
-    },
-  });
-}
+    });
+  },
+});
+
 if (verbose) {
   dataCache.setDebugLogger(console.log);
 }

diff --git a/src/common/runtime/standalone.ts b/src/common/runtime/standalone.ts
@@ -2,7 +2,7 @@
 /* eslint no-console: "off" */
 
 import { dataCache } from '../framework/data_cache.js';
-import { setBaseResourcePath } from '../framework/resources.js';
+import { getResourcePath, setBaseResourcePath } from '../framework/resources.js';
 import { globalTestConfig } from '../framework/test_config.js';
 import { DefaultTestFileLoader } from '../internal/file_loader.js';
 import { Logger } from '../internal/logging/logger.js';
@@ -80,7 +80,7 @@ if (powerPreference || compatibility) {
 
 dataCache.setStore({
   load: async (path: string) => {
-    const response = await fetch(`data/${path}`);
+    const response = await fetch(getResourcePath(`cache/${path}`));
     if (!response.ok) {
       return Promise.reject(response.statusText);
     }