Remove compression on the case cache, split files to reduce transitiv…

…e deps (#3197)

* Remove compression on the case cache

Compression is not stable between Node versions, nor between operating systems.

This inflates the cache size 10x (from 4.3M to 49M), however these files should not be frequently changing, and Git should be performing compressions with packfiles and during network operations.

* Split cache generation from spec.ts files

* Breakup monolithic imports

Move commonly used code out of monolithic imports to reduce the amount of code imported by the `.cache.ts` files.

Add a new `MUST_NOT_BE_IMPORTED_BY_DATA_CACHE` token that can be used to figure out why the cache wants to regenerate for a given file. Add this to device_pool.ts, as this really shouldn't be depended on for the cache.

docs/case_cache.md

@@ -9,8 +9,8 @@ for multiple CTS runs.
 ## File cache
 
 To speed up execution of the CTS, the CTS git repo holds holds pre-computed
-test cases, serialized in a set of gzip-compressed binary files under
-[`src/resources/cache`](../src/resources/cache).
+test cases, generated from `*.cache.ts` files and serialized in a set of binary
+files under [`src/resources/cache`](../src/resources/cache).
 
 These files are regenerated by [`src/common/tools/gen_cache.ts`](../src/common/tools/gen_cache.ts)
 which can be run with `npx grunt run:generate-cache`.
@@ -24,10 +24,11 @@ output, and compares this hash to the hash stored in
 [`src/resources/cache/hashes.json`](`../src/resources/cache/hashes.json`). Only
 those cache files with differing hashes are rebuilt.
 
-Since source changes will sometimes change the hash without changing the generated cache,
-sometimes the cache will be regenerated unnecessarily. **This is OK, but try to avoid committing
-no-op regenerations - this will happen if your version of Node produces different gzip outputs
-than the original committer's Node did for the same input.**
+Transitive imports easily grow, and these can cause unnecessary rebuilds of the cache.
+To help avoid unnecessary rebuilds, files that are known to not be used by the cache can be
+annotated with a `MUST_NOT_BE_IMPORTED_BY_DATA_CACHE` comment anywhere in the file. If a file with
+this comment is transitively imported by a `.cache.ts` file, then the cache generation tool will
+error with a trace of the imports from the `.cache.ts` file to the file with this comment.
 
 The cache files are copied from [`src/resources/cache`](../src/resources/cache)
 to the `resources/cache` subdirectory of the
@@ -46,13 +47,15 @@ will build the cases during CTS execution and store the results in an in-memory
 
 To add test cases to the cache:
 
-1. Import `makeCaseCache` from [`'case_cache.js'`](../src/webgpu/shader/execution/expression/case_cache.ts);
+1. Create a new <code><i>my_file</i>.cache.ts</code> file.
+
+2. In that file, import `makeCaseCache` from [`'case_cache.js'`](../src/webgpu/shader/execution/expression/case_cache.ts);
 
 ```ts
 import { makeCaseCache } from '../case_cache.js'; // your relative path may vary
 ```
 
-2. Declare an exported global variable with the name `d`, assigned with the return value of `makeCaseCache()`:
+3. Declare an exported global variable with the name `d`, assigned with the return value of `makeCaseCache()`:
 
 ```ts
 export const d = makeCaseCache('unique/path/of/your/cache/file', {
@@ -65,12 +68,14 @@ export const d = makeCaseCache('unique/path/of/your/cache/file', {
 });
 ```
 
-3. To load the cases from the cache, use `d.get();`
+4. To use the cached cases in a <code><i>my_file</i>.spec.ts</code> file, import `d` from <code><i>my_file</i>.cache.js</code>, and use `d.get();`
 
 ```ts
+import { d } from './my_file.cache.js';
+
 const cases = await d.get('name_of_your_case');
 // cases will either be loaded from the cache file, loaded from the in-memory
 // LRU, or built on the fly.
 ```
 
-4. Run `npx grunt run generate-cache` to generate the new cache file.
+5. Run `npx grunt run generate-cache` to generate the new cache file.

src/common/framework/data_cache.ts

@@ -187,11 +187,11 @@ export interface Cacheable<Data> {
   /**
    * serialize() encodes `data` to a binary representation so that it can be stored in a cache file.
    */
-  serialize(data: Data): Promise<Uint8Array>;
+  serialize(data: Data): Uint8Array;
 
   /**
    * deserialize() is the inverse of serialize(), decoding the binary representation back to a Data
    * object.
    */
-  deserialize(binary: Uint8Array): Promise<Data>;
+  deserialize(binary: Uint8Array): Data;
 }

src/common/tools/gen_cache.ts

@@ -89,7 +89,7 @@ dataCache.setStore({
 });
 setIsBuildingDataCache();
 
-const specFileSuffix = __filename.endsWith('.ts') ? '.spec.ts' : '.spec.js';
+const cacheFileSuffix = __filename.endsWith('.ts') ? '.cache.ts' : '.cache.js';
 
 /**
  * @returns a list of all the files under 'dir' that has the given extension
@@ -105,13 +105,27 @@ function glob(dir: string, ext: string) {
         files.push(`${file}/${child}`);
       }
     }
+
     if (path.endsWith(ext) && fs.statSync(path).isFile()) {
       files.push(file);
     }
   }
   return files;
 }
 
+/**
+ * Exception type thrown by SourceHasher.hashFile() when a file annotated with
+ * MUST_NOT_BE_IMPORTED_BY_DATA_CACHE is transitively imported by a .cache.ts file.
+ */
+class InvalidImportException {
+  constructor(path: string) {
+    this.stack = [path];
+  }
+  toString(): string {
+    return `invalid transitive import for cache:\n  ${this.stack.join('\n  ')}`;
+  }
+  readonly stack: string[];
+}
 /**
  * SourceHasher is a utility for producing a hash of a source .ts file and its imported source files.
  */
@@ -141,8 +155,19 @@ class SourceHasher {
     const normalized = content.replace('\r\n', '\n');
     let hash = crc32(normalized);
     for (const importPath of parseImports(path, normalized)) {
-      const importHash = this.hashFile(importPath);
-      hash = this.hashCombine(hash, importHash);
+      try {
+        const importHash = this.hashFile(importPath);
+        hash = this.hashCombine(hash, importHash);
+      } catch (ex) {
+        if (ex instanceof InvalidImportException) {
+          ex.stack.push(path);
+          throw ex;
+        }
+      }
+    }
+
+    if (content.includes('MUST_NOT_BE_IMPORTED_BY_DATA_CACHE')) {
+      throw new InvalidImportException(path);
     }
 
     this.hashes.set(path, hash);
@@ -180,7 +205,7 @@ async function build(suiteDir: string) {
   }
 
   // Crawl files and convert paths to be POSIX-style, relative to suiteDir.
-  const filesToEnumerate = glob(suiteDir, specFileSuffix)
+  const filesToEnumerate = glob(suiteDir, cacheFileSuffix)
     .map(p => `${suiteDir}/${p}`)
     .sort();
 
@@ -221,7 +246,7 @@ and
           }
           console.log(`building '${outPath}'`);
           const data = await cacheable.build();
-          const serialized = await cacheable.serialize(data);
+          const serialized = cacheable.serialize(data);
           fs.mkdirSync(path.dirname(outPath), { recursive: true });
           fs.writeFileSync(outPath, serialized, 'binary');
           fileHashes[cacheable.path] = fileHash;

src/resources/cache/hashes.json

@@ -1,106 +1,106 @@
 {
-  "webgpu/shader/execution/binary/af_addition.bin": "8e47fc86",
-  "webgpu/shader/execution/binary/af_logical.bin": "348bc279",
-  "webgpu/shader/execution/binary/af_division.bin": "82029c55",
-  "webgpu/shader/execution/binary/af_matrix_addition.bin": "6deae35e",
-  "webgpu/shader/execution/binary/af_matrix_subtraction.bin": "221a913c",
-  "webgpu/shader/execution/binary/af_multiplication.bin": "53c87c5b",
-  "webgpu/shader/execution/binary/af_remainder.bin": "dadb3ceb",
-  "webgpu/shader/execution/binary/af_subtraction.bin": "a0a71e32",
-  "webgpu/shader/execution/binary/f16_addition.bin": "acfee8",
-  "webgpu/shader/execution/binary/f16_logical.bin": "78e0eec8",
-  "webgpu/shader/execution/binary/f16_division.bin": "289b2aba",
-  "webgpu/shader/execution/binary/f16_matrix_addition.bin": "bada3cbe",
-  "webgpu/shader/execution/binary/f16_matrix_matrix_multiplication.bin": "bce23630",
-  "webgpu/shader/execution/binary/f16_matrix_scalar_multiplication.bin": "2d6bcf91",
-  "webgpu/shader/execution/binary/f16_matrix_subtraction.bin": "c3db5537",
-  "webgpu/shader/execution/binary/f16_matrix_vector_multiplication.bin": "402931c",
-  "webgpu/shader/execution/binary/f16_multiplication.bin": "ce381066",
-  "webgpu/shader/execution/binary/f16_remainder.bin": "b2bd9246",
-  "webgpu/shader/execution/binary/f16_subtraction.bin": "7f7e9abc",
-  "webgpu/shader/execution/binary/f32_addition.bin": "71302f1a",
-  "webgpu/shader/execution/binary/f32_logical.bin": "ad34883d",
-  "webgpu/shader/execution/binary/f32_division.bin": "b35602bb",
-  "webgpu/shader/execution/binary/f32_matrix_addition.bin": "82339c4",
-  "webgpu/shader/execution/binary/f32_matrix_matrix_multiplication.bin": "33c2bd83",
-  "webgpu/shader/execution/binary/f32_matrix_scalar_multiplication.bin": "6b6dd016",
-  "webgpu/shader/execution/binary/f32_matrix_subtraction.bin": "7733299f",
-  "webgpu/shader/execution/binary/f32_matrix_vector_multiplication.bin": "d99b4cc3",
-  "webgpu/shader/execution/binary/f32_multiplication.bin": "ff22107c",
-  "webgpu/shader/execution/binary/f32_remainder.bin": "44133c4c",
-  "webgpu/shader/execution/binary/f32_subtraction.bin": "a9a5a319",
-  "webgpu/shader/execution/binary/i32_arithmetic.bin": "eb21bd8f",
-  "webgpu/shader/execution/binary/i32_comparison.bin": "83121ec4",
-  "webgpu/shader/execution/binary/u32_arithmetic.bin": "8d1429d1",
-  "webgpu/shader/execution/binary/u32_comparison.bin": "3491ba55",
-  "webgpu/shader/execution/abs.bin": "a5dcabd0",
-  "webgpu/shader/execution/acos.bin": "adbccc79",
-  "webgpu/shader/execution/acosh.bin": "7c07ba41",
-  "webgpu/shader/execution/asin.bin": "8fc62434",
-  "webgpu/shader/execution/asinh.bin": "a00937c0",
-  "webgpu/shader/execution/atan.bin": "a237eaa6",
-  "webgpu/shader/execution/atan2.bin": "b41d74be",
-  "webgpu/shader/execution/atanh.bin": "2b6934d5",
-  "webgpu/shader/execution/bitcast.bin": "86e79d18",
-  "webgpu/shader/execution/ceil.bin": "42a99291",
-  "webgpu/shader/execution/clamp.bin": "b3fc754f",
-  "webgpu/shader/execution/cos.bin": "13305519",
-  "webgpu/shader/execution/cosh.bin": "20965093",
-  "webgpu/shader/execution/cross.bin": "3cfeb2a1",
-  "webgpu/shader/execution/degrees.bin": "6d9b3197",
-  "webgpu/shader/execution/determinant.bin": "d09e35b7",
-  "webgpu/shader/execution/distance.bin": "790d45ea",
-  "webgpu/shader/execution/dot.bin": "5f7aa2bc",
-  "webgpu/shader/execution/exp.bin": "932473",
-  "webgpu/shader/execution/exp2.bin": "1ada59d6",
-  "webgpu/shader/execution/faceForward.bin": "dc44d720",
-  "webgpu/shader/execution/floor.bin": "cbbe2d21",
-  "webgpu/shader/execution/fma.bin": "b5f22cac",
-  "webgpu/shader/execution/fract.bin": "b9b72ee2",
-  "webgpu/shader/execution/frexp.bin": "cc49a118",
-  "webgpu/shader/execution/inverseSqrt.bin": "ab8d39ed",
-  "webgpu/shader/execution/ldexp.bin": "cca23df2",
-  "webgpu/shader/execution/length.bin": "4693ba28",
-  "webgpu/shader/execution/log.bin": "75d7c534",
-  "webgpu/shader/execution/log2.bin": "c0c65cb0",
-  "webgpu/shader/execution/max.bin": "11d13c22",
-  "webgpu/shader/execution/min.bin": "5c701467",
-  "webgpu/shader/execution/mix.bin": "af998787",
-  "webgpu/shader/execution/modf.bin": "3a3b367a",
-  "webgpu/shader/execution/normalize.bin": "ce31e122",
-  "webgpu/shader/execution/pack2x16float.bin": "d3198def",
-  "webgpu/shader/execution/pow.bin": "847bf3ff",
-  "webgpu/shader/execution/quantizeToF16.bin": "e2f5c3f9",
-  "webgpu/shader/execution/radians.bin": "8869a385",
-  "webgpu/shader/execution/reflect.bin": "2be62e10",
-  "webgpu/shader/execution/refract.bin": "a95cf2a",
-  "webgpu/shader/execution/round.bin": "cdc5ae01",
-  "webgpu/shader/execution/saturate.bin": "bb20b07c",
-  "webgpu/shader/execution/sign.bin": "a59f6a2b",
-  "webgpu/shader/execution/sin.bin": "b36c7552",
-  "webgpu/shader/execution/sinh.bin": "9385405",
-  "webgpu/shader/execution/smoothstep.bin": "5d740ae7",
-  "webgpu/shader/execution/sqrt.bin": "5e42a89b",
-  "webgpu/shader/execution/step.bin": "4447b38c",
-  "webgpu/shader/execution/tan.bin": "b47dfc33",
-  "webgpu/shader/execution/tanh.bin": "8bd0ae74",
-  "webgpu/shader/execution/transpose.bin": "afbbaef0",
-  "webgpu/shader/execution/trunc.bin": "4d15f320",
-  "webgpu/shader/execution/unpack2x16float.bin": "40ca16cc",
-  "webgpu/shader/execution/unpack2x16snorm.bin": "39ece93e",
-  "webgpu/shader/execution/unpack2x16unorm.bin": "dbdc06b8",
-  "webgpu/shader/execution/unpack4x8snorm.bin": "c83bc6c2",
-  "webgpu/shader/execution/unpack4x8unorm.bin": "678c5245",
-  "webgpu/shader/execution/unary/af_arithmetic.bin": "1679466",
-  "webgpu/shader/execution/unary/af_assignment.bin": "202b8019",
-  "webgpu/shader/execution/unary/bool_conversion.bin": "6cb7e11a",
-  "webgpu/shader/execution/unary/f16_arithmetic.bin": "d0409431",
-  "webgpu/shader/execution/unary/f16_conversion.bin": "884688a",
-  "webgpu/shader/execution/unary/f32_arithmetic.bin": "8925bac7",
-  "webgpu/shader/execution/unary/f32_conversion.bin": "798aaba2",
-  "webgpu/shader/execution/unary/i32_arithmetic.bin": "c21ece3",
-  "webgpu/shader/execution/unary/i32_complement.bin": "f65f0bec",
-  "webgpu/shader/execution/unary/i32_conversion.bin": "7ce468a1",
-  "webgpu/shader/execution/unary/u32_complement.bin": "d6408865",
-  "webgpu/shader/execution/unary/u32_conversion.bin": "aa09fe0"
+  "webgpu/shader/execution/binary/af_addition.bin": "b5345cca",
+  "webgpu/shader/execution/binary/af_logical.bin": "9ec85311",
+  "webgpu/shader/execution/binary/af_division.bin": "23dd840e",
+  "webgpu/shader/execution/binary/af_matrix_addition.bin": "ee95539e",
+  "webgpu/shader/execution/binary/af_matrix_subtraction.bin": "a2e64d57",
+  "webgpu/shader/execution/binary/af_multiplication.bin": "94d7a574",
+  "webgpu/shader/execution/binary/af_remainder.bin": "79f5abc4",
+  "webgpu/shader/execution/binary/af_subtraction.bin": "62dce5bf",
+  "webgpu/shader/execution/binary/f16_addition.bin": "ab3b4d9c",
+  "webgpu/shader/execution/binary/f16_logical.bin": "1edd08ec",
+  "webgpu/shader/execution/binary/f16_division.bin": "849eed26",
+  "webgpu/shader/execution/binary/f16_matrix_addition.bin": "54afc3f8",
+  "webgpu/shader/execution/binary/f16_matrix_matrix_multiplication.bin": "df5d3ccc",
+  "webgpu/shader/execution/binary/f16_matrix_scalar_multiplication.bin": "3be6032e",
+  "webgpu/shader/execution/binary/f16_matrix_subtraction.bin": "98a29837",
+  "webgpu/shader/execution/binary/f16_matrix_vector_multiplication.bin": "c463d7e0",
+  "webgpu/shader/execution/binary/f16_multiplication.bin": "cdf0775d",
+  "webgpu/shader/execution/binary/f16_remainder.bin": "e82ef89f",
+  "webgpu/shader/execution/binary/f16_subtraction.bin": "5125b5b0",
+  "webgpu/shader/execution/binary/f32_addition.bin": "d3b8004",
+  "webgpu/shader/execution/binary/f32_logical.bin": "fab7cfc5",
+  "webgpu/shader/execution/binary/f32_division.bin": "e9fdb0a9",
+  "webgpu/shader/execution/binary/f32_matrix_addition.bin": "d1165469",
+  "webgpu/shader/execution/binary/f32_matrix_matrix_multiplication.bin": "45a79521",
+  "webgpu/shader/execution/binary/f32_matrix_scalar_multiplication.bin": "bf8da7d0",
+  "webgpu/shader/execution/binary/f32_matrix_subtraction.bin": "82d7fa9",
+  "webgpu/shader/execution/binary/f32_matrix_vector_multiplication.bin": "57eebe84",
+  "webgpu/shader/execution/binary/f32_multiplication.bin": "eb0baa56",
+  "webgpu/shader/execution/binary/f32_remainder.bin": "e6059990",
+  "webgpu/shader/execution/binary/f32_subtraction.bin": "d57cfa1",
+  "webgpu/shader/execution/binary/i32_arithmetic.bin": "de93ee2a",
+  "webgpu/shader/execution/binary/i32_comparison.bin": "aaa1f21b",
+  "webgpu/shader/execution/binary/u32_arithmetic.bin": "d79a1011",
+  "webgpu/shader/execution/binary/u32_comparison.bin": "31764c75",
+  "webgpu/shader/execution/abs.bin": "9278901c",
+  "webgpu/shader/execution/acos.bin": "9c77ebf9",
+  "webgpu/shader/execution/acosh.bin": "2baaf7cc",
+  "webgpu/shader/execution/asin.bin": "8fd7a678",
+  "webgpu/shader/execution/asinh.bin": "321a7294",
+  "webgpu/shader/execution/atan.bin": "d3dada7e",
+  "webgpu/shader/execution/atan2.bin": "2871596c",
+  "webgpu/shader/execution/atanh.bin": "e7158ef",
+  "webgpu/shader/execution/bitcast.bin": "aa7d6f0e",
+  "webgpu/shader/execution/ceil.bin": "3c1d91ad",
+  "webgpu/shader/execution/clamp.bin": "a53fdbda",
+  "webgpu/shader/execution/cos.bin": "73b83288",
+  "webgpu/shader/execution/cosh.bin": "39ca32a4",
+  "webgpu/shader/execution/cross.bin": "409c3e26",
+  "webgpu/shader/execution/degrees.bin": "a44bb2b1",
+  "webgpu/shader/execution/determinant.bin": "41a06528",
+  "webgpu/shader/execution/distance.bin": "fe3ea84f",
+  "webgpu/shader/execution/dot.bin": "f70abfec",
+  "webgpu/shader/execution/exp.bin": "7587e11b",
+  "webgpu/shader/execution/exp2.bin": "c6f34d2b",
+  "webgpu/shader/execution/faceForward.bin": "8eb57a7d",
+  "webgpu/shader/execution/floor.bin": "b9ea647b",
+  "webgpu/shader/execution/fma.bin": "b9f657b2",
+  "webgpu/shader/execution/fract.bin": "73a18e4f",
+  "webgpu/shader/execution/frexp.bin": "857fe9b7",
+  "webgpu/shader/execution/inverseSqrt.bin": "a595983a",
+  "webgpu/shader/execution/ldexp.bin": "a04aaeca",
+  "webgpu/shader/execution/length.bin": "695740fb",
+  "webgpu/shader/execution/log.bin": "6703ec1a",
+  "webgpu/shader/execution/log2.bin": "8339559d",
+  "webgpu/shader/execution/max.bin": "ec000a56",
+  "webgpu/shader/execution/min.bin": "156cf7cc",
+  "webgpu/shader/execution/mix.bin": "32064d21",
+  "webgpu/shader/execution/modf.bin": "a5003ce0",
+  "webgpu/shader/execution/normalize.bin": "c12bac96",
+  "webgpu/shader/execution/pack2x16float.bin": "b2cb12ea",
+  "webgpu/shader/execution/pow.bin": "ee87eccb",
+  "webgpu/shader/execution/quantizeToF16.bin": "f77ae7e3",
+  "webgpu/shader/execution/radians.bin": "7ecbe5be",
+  "webgpu/shader/execution/reflect.bin": "d37034bd",
+  "webgpu/shader/execution/refract.bin": "6dc9adcf",
+  "webgpu/shader/execution/round.bin": "d91faa0f",
+  "webgpu/shader/execution/saturate.bin": "93230980",
+  "webgpu/shader/execution/sign.bin": "fb1071b8",
+  "webgpu/shader/execution/sin.bin": "a9ed8361",
+  "webgpu/shader/execution/sinh.bin": "65ba80fc",
+  "webgpu/shader/execution/smoothstep.bin": "17e1e103",
+  "webgpu/shader/execution/sqrt.bin": "4c29a5d4",
+  "webgpu/shader/execution/step.bin": "aed08458",
+  "webgpu/shader/execution/tan.bin": "5f671594",
+  "webgpu/shader/execution/tanh.bin": "991e903a",
+  "webgpu/shader/execution/transpose.bin": "9d388797",
+  "webgpu/shader/execution/trunc.bin": "959fe8bc",
+  "webgpu/shader/execution/unpack2x16float.bin": "26e5b05e",
+  "webgpu/shader/execution/unpack2x16snorm.bin": "c756d8a3",
+  "webgpu/shader/execution/unpack2x16unorm.bin": "5257591a",
+  "webgpu/shader/execution/unpack4x8snorm.bin": "83fbd41a",
+  "webgpu/shader/execution/unpack4x8unorm.bin": "77d46acb",
+  "webgpu/shader/execution/unary/af_arithmetic.bin": "2edb2dc5",
+  "webgpu/shader/execution/unary/af_assignment.bin": "36c04bba",
+  "webgpu/shader/execution/unary/bool_conversion.bin": "cb53bf65",
+  "webgpu/shader/execution/unary/f16_arithmetic.bin": "9f459fc4",
+  "webgpu/shader/execution/unary/f16_conversion.bin": "bf055ca9",
+  "webgpu/shader/execution/unary/f32_arithmetic.bin": "e15c49e7",
+  "webgpu/shader/execution/unary/f32_conversion.bin": "92e5069f",
+  "webgpu/shader/execution/unary/i32_arithmetic.bin": "d322b73d",
+  "webgpu/shader/execution/unary/i32_complement.bin": "c4e6cbb",
+  "webgpu/shader/execution/unary/i32_conversion.bin": "d6905a0f",
+  "webgpu/shader/execution/unary/u32_complement.bin": "d0e4aa97",
+  "webgpu/shader/execution/unary/u32_conversion.bin": "5a96b263"
 }