diff --git a/.eslintrc.json b/.eslintrc.json index f99f65896007..3ee5d66e3c3a 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -14,7 +14,9 @@ } ], "linebreak-style": ["error", "unix"], + "object-shorthand": "error", "no-console": "error", + "no-useless-rename": "error", "@typescript-eslint/no-inferrable-types": "off", "import/order": [ "error", diff --git a/Gruntfile.js b/Gruntfile.js index 243b4e7f2732..061d352f2a41 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -23,7 +23,7 @@ module.exports = function (grunt) { }, test: { cmd: 'node', - args: ['tools/run', 'unittests:'], + args: ['tools/run', 'unittests:*'], }, 'build-out': { cmd: 'node', diff --git a/docs/terms.md b/docs/terms.md index d718c648fbac..be9d83a15eef 100644 --- a/docs/terms.md +++ b/docs/terms.md @@ -1,11 +1,36 @@ # Writing Tests -- _Suites_ contain multiple: - - _READMEs_ - - _Test Spec Files_. Contains one: - - _Test Group_. Defines a _test fixture_ and contains multiple: - - _Tests_. Defines a _test function_ and contains multiple: - - _Test Cases_, each with the same test function but different _parameters_. +Each test suite is organized as a tree, both in the filesystem and further within each file. + +- _Suites_, e.g. `src/webgpu/`. + - _READMEs_, e.g. `src/webgpu/README.txt`. + - _Test Spec Files_, e.g. `src/webgpu/examples.spec.ts`. + Identified by their file path. + Each test spec file provides a description and a _Test Group_. + A _Test Group_ defines a test fixture, and contains multiple: + - _Tests_. + Identified by a comma-separated list of parts (e.g. `basic,async`) + which define a path through a filesystem-like tree (analogy: `basic/async.txt`). + Defines a _test function_ and contains multiple: + - _Test Cases_. + Identified by a list of _Public Parameters_ (e.g. `x` = `1`, `y` = `2`). + Each Test Case has the same test function but different Public Parameters. + +## Test Tree + +A _Test Tree_ is a tree whose leaves are individual Test Cases. + +A Test Tree can be thought of as follows: + +- Suite, which is the root of a tree with "leaves" which are: + - Test Spec Files, each of which is a tree with "leaves" which are: + - Tests, each of which is a tree with leaves which are: + - Test Cases. + +(In the implementation, this conceptual tree of trees is decomposed into one big tree +whose leaves are Test Cases.) + +**Type:** `TestTree` ## Suite @@ -22,30 +47,53 @@ Each member of a suite is identified by its path within the suite. Describes (in prose) the contents of a subdirectory in a suite. -**Type:** After a README is loaded, it is stored as a `ReadmeFile`. +READMEs are only processed at build time, when generating the _Listing_ for a suite. -## IDs +**Type:** `TestSuiteListingEntryReadme` -### Test Spec ID +## Queries -Uniquely identifies a single test spec file. -Comprised of suite name (`suite`) and test spec file path relative to the suite root (`path`). +A _Query_ is a structured object which specifies a subset of cases in exactly one Suite. +A Query can be represented uniquely as a string. +Queries are used to: -**Type:** `TestSpecID` +- Identify a subtree of a suite (by identifying the root node of that subtree). +- Identify individual cases. +- Represent the list of tests that a test runner (standalone, wpt, or cmdline) should run. +- Identify subtrees which should not be "collapsed" during WPT `cts.html` generation, + so that that cts.html "variants" can have individual test expectations + (i.e. marked as "expected to fail", "skip", etc.). -**Example:** `{ suite: 'webgpu', path: 'command_buffer/compute/basic' }` corresponds to -`src/webgpu/command_buffer/compute/basic.spec.ts`. +There are four types of `TestQuery`: -### Test Case ID +- `TestQueryMultiFile` represents any subtree of the file hierarchy: + - `suite:*` + - `suite:path,to,*` + - `suite:path,to,file,*` +- `TestQueryMultiTest` represents any subtree of the test hierarchy: + - `suite:path,to,file:*` + - `suite:path,to,file:path,to,*` + - `suite:path,to,file:path,to,test,*` +- `TestQueryMultiCase` represents any subtree of the case hierarchy: + - `suite:path,to,file:path,to,test:*` + - `suite:path,to,file:path,to,test:my=0;*` + - `suite:path,to,file:path,to,test:my=0;params="here";*` +- `TestQuerySingleCase` represents as single case: + - `suite:path,to,file:path,to,test:my=0;params="here"` -Uniquely identifies a single test case within a test spec. +Test Queries are a **weakly ordered set**: any query is +_Unordered_, _Equal_, _StrictSuperset_, or _StrictSubset_ relative to any other. +This property is used to construct the complete tree of test cases. +In the examples above, every example query is a StrictSubset of the previous one +(note: even `:*` is a subset of `,*`). -Comprised of test name (`test`) and the parameters for a case (`params`). -(If `params` is null, there is only one case for the test.) +In the WPT and standalone harnesses, the query is stored in the URL, e.g. +`index.html?q=q:u,e:r,y:*`. -**Type:** `TestCaseID` +Queries are selectively URL-encoded for readability and compatibility with browsers +(see `encodeURIComponentSelectively`). -**Example:** `{ test: '', params: { 'value': 1 } }` +**Type:** `TestQuery` ## Listing @@ -53,39 +101,35 @@ A listing of the **test spec files** in a suite. This can be generated only in Node, which has filesystem access (see `src/tools/crawl.ts`). As part of the build step, a _listing file_ is generated (see `src/tools/gen.ts`) so that the -test files can be discovered by the web runner (since it does not have filesystem access). +Test Spec Files can be discovered by the web runner (since it does not have filesystem access). **Type:** `TestSuiteListing` ### Listing File -**Example:** `out/webgpu/listing.js` +Each Suite has one Listing File (`suite/listing.[tj]s`), containing a list of the files +in the suite. -## Test Spec +In `src/suite/listing.ts`, this is computed dynamically. +In `out/suite/listing.js`, the listing has been pre-baked (by `tools/gen_listings`). -May be either a _test spec file_ or a _filtered test spec_. -It is identified by a `TestSpecID`. +**Type:** Once `import`ed, `ListingFile` -Always contains one `RunCaseIterable`. +**Example:** `out/webgpu/listing.js` -**Type:** `TestSpec` +## Test Spec File -### Test Spec File +A Test Spec File has a `description` and a Test Group (under which tests and cases are defined). -A single `.spec.ts` file. It always contains one _test group_. +**Type:** Once `import`ed, `SpecFile` **Example:** `src/webgpu/**/*.spec.ts` -### Filtered Test Spec - -A subset of the tests in a single test spec file. -It is created at runtime, via a filter. - -It contains one "virtual" test group, which has type `RunCaseIterable`. +## Test Group -## Test Group / Group +A subtree of tests. There is one Test Group per Test Spec File. -A collection of test cases. There is one per test spec file. +The Test Fixture used for tests is defined at TestGroup creation. **Type:** `TestGroup` @@ -93,18 +137,17 @@ A collection of test cases. There is one per test spec file. One test. It has a single _test function_. -It may represent multiple _test cases_, each of which runs the same test function with different **parameters**. +It may represent multiple _test cases_, each of which runs the same Test Function with different +Parameters. -It is created by `TestGroup.test()`. -At creation time, it can be parameterized via `Test.params()`. - -**Type:** `Test` +A test is named using `TestGroup.test()`, which returns a `TestBuilder`. +`TestBuilder.params()` can optionally be used to parameterize the test. +Then, `TestBuilder.fn()` provides the Test Function. ### Test Function -A test function is defined inline inside of a `TestGroup.test()` call. - -It receives an instance of the appropriate _test fixture_, through which it produce test results. +When a test case is run, the Test Function receives an instance of the +Test Fixture provided to the Test Group, producing test results. **Type:** `TestFn` @@ -116,9 +159,20 @@ A single case of a test. It is identified by a `TestCaseID`: a test name, and it ## Parameters / Params +Each Test Case has a (possibly empty) set of Parameters. +The parameters are available to the Test Function `f(t)` via `t.params`. + +A set of Public Parameters identifies a Test Case within a Test. + +There are also Private Paremeters: any parameter name beginning with an underscore (`_`). +These parameters are not part of the Test Case identification, but are still passed into +the Test Function. They can be used to manually specify expected results. + +**Type:** `CaseParams` + ## Test Fixture / Fixture -Test fixtures provide helpers for tests to use. +_Test Fixtures_ provide helpers for tests to use. A new instance of the fixture is created for every run of every test case. There is always one fixture class for a whole test group (though this may change). @@ -126,7 +180,7 @@ There is always one fixture class for a whole test group (though this may change The fixture is also how a test gets access to the _case recorder_, which allows it to produce test results. -They are also how tests produce results: `.log()`, `.fail()`, etc. +They are also how tests produce results: `.skip()`, `.fail()`, etc. **Type:** `Fixture` @@ -138,137 +192,61 @@ Provides basic fixture utilities most useful in the `unittests` suite. Provides utilities useful in WebGPU CTS tests. -# Running Tests - -- _Queries_ contain multiple: - - _Filters_ (positive or negative). - -## Query - -A query is a string which denotes a subset of a test suite to run. -It is comprised of a list of filters. -A case is included in the subset if it matches any of the filters. - -In the WPT and standalone harnesses, the query is stored in the URL. - -Queries are selectively URL-encoded for readability and compatibility with browsers. - -**Example:** `?q=unittests:param_helpers:combine/=` -**Example:** `?q=unittests:param_helpers:combine/mixed=` - -**Type:** None yet. TODO - -### Filter - -A filter matches a set of cases in a suite. - -Each filter may match one of: - -- `S:s` In one suite `S`, all specs whose paths start with `s` (which may be empty). -- `S:s:t` In one spec `S:s`, all tests whose names start with `t` (which may be empty). -- `S:s:t~c` In one test `S:s:t`, all cases whose params are a superset of `c`. -- `S:s:t=c` In one test `S:s:t`, the single case whose params equal `c` (empty string = `{}`). - -**Type:** `TestFilter` - -### Using filters to split expectations - -A set of cases can be split using negative filters. For example, imagine you have one WPT test variant: - -- `webgpu/cts.html?q=unittests:param_helpers:` - -But one of the cases is failing. To be able to suppress the failing test without losing test coverage, the WPT test variant can be split into two variants: - -- `webgpu/cts.html?q=unittests:param_helpers:¬=unittests:param_helpers:combine/mixed:` -- `webgpu/cts.html?q=unittests:param_helpers:combine/mixed:` - -This runs the same set of cases, but in two separate page loads. - # Test Results ## Logger A logger logs the results of a whole test run. -It saves an empty `LiveTestSpecResult` into its results array, then creates a +It saves an empty `LiveTestSpecResult` into its results map, then creates a _test spec recorder_, which records the results for a group into the `LiveTestSpecResult`. -### Test Spec Recorder - -Refers to a `LiveTestSpecResult` in the logger, and writes results into it. - -It saves an empty `LiveTestCaseResult` into its `LiveTestSpecResult`, then creates a -_test case recorder_, which records the results for a case into the `LiveTestCaseResult`. +**Type:** `Logger` ### Test Case Recorder -Records the actual results of running a test case (its pass-status, run time, and logs) -into its `LiveTestCaseResult`. +Refers to a `LiveTestCaseResult` created by the logger. +Records the results of running a test case (its pass-status, run time, and logs) into it. + +**Types:** `TestCaseRecorder`, `LiveTestCaseResult` #### Test Case Status The `status` of a `LiveTestCaseResult` can be one of: -- `'running'` -- `'fail'` -- `'warn'` +- `'running'` (only while still running) - `'pass'` +- `'skip'` +- `'warn'` +- `'fail'` + +The "worst" result from running a case is always reported (fail > warn > skip > pass). +Note this means a test can still fail if it's "skipped", if it failed before +`.skip()` was called. -The "worst" result from running a case is always reported. +**Type:** `Status` ## Results Format The results are returned in JSON format. -They are designed to be easily merged in JavaScript. -(TODO: Write a merge tool.) +They are designed to be easily merged in JavaScript: +the `"results"` can be passed into the constructor of `Map` and merged from there. -(TODO: Update these docs if the format changes.) +(TODO: Write a merge tool, if needed.) ```js { - "version": "e24c459b46c4815f93f0aed5261e28008d1f2882-dirty", + "version": "bf472c5698138cdf801006cd400f587e9b1910a5-dirty", "results": [ - // LiveTestSpecResult objects - { - "spec": "unittests:basic:", - "cases": [ - // LiveTestCaseResult objects - { - "test": "test/sync", - "params": null, - "status": "pass", - "timems": 145, - "logs": [] - }, - { - "test": "test/async", - "params": null, - "status": "pass", - "timems": 26, - "logs": [] - } - ] - }, - { - "spec": "unittests:test_group:", - "cases": [ - { - "test": "invalid test name", - "params": { "char": "\"" }, - "status": "pass", - "timems": 66, - "logs": ["OK: threw"] - }, - { - "test": "invalid test name", - "params": { "char": "`" }, - "status": "pass", - "timems": 15, - "logs": ["OK: threw"] - } - ] - } + [ + "unittests:async_mutex:basic:", + { "status": "pass", "timems": 0.286, "logs": [] } + ], + [ + "unittests:async_mutex:serial:", + { "status": "pass", "timems": 0.415, "logs": [] } + ] ] } ``` diff --git a/prettier.config.js b/prettier.config.js index 5b67ab5f64a9..9f4053f719e7 100644 --- a/prettier.config.js +++ b/prettier.config.js @@ -1,8 +1,8 @@ module.exports = { printWidth: 100, + arrowParens: 'avoid', bracketSpacing: true, singleQuote: true, trailingComma: 'es5', - arrowParens: 'avoid', }; diff --git a/src/common/framework/allowed_characters.ts b/src/common/framework/allowed_characters.ts deleted file mode 100644 index bc54c2c73954..000000000000 --- a/src/common/framework/allowed_characters.ts +++ /dev/null @@ -1,2 +0,0 @@ -// It may be OK to add more allowed characters here. -export const allowedTestNameCharacters = 'a-zA-Z0-9/_'; diff --git a/src/common/framework/file_loader.ts b/src/common/framework/file_loader.ts new file mode 100644 index 000000000000..538e3dbe418d --- /dev/null +++ b/src/common/framework/file_loader.ts @@ -0,0 +1,50 @@ +import { parseQuery } from './query/parseQuery.js'; +import { RunCaseIterable } from './test_group.js'; +import { TestSuiteListing } from './test_suite_listing.js'; +import { loadTreeForQuery, TestTree, TestTreeLeaf } from './tree.js'; + +// A listing file, e.g. either of: +// - `src/webgpu/listing.ts` (which is dynamically computed, has a Promise) +// - `out/webgpu/listing.js` (which is pre-baked, has a TestSuiteListing) +interface ListingFile { + listing: Promise | TestSuiteListing; +} + +// A .spec.ts file, as imported. +export interface SpecFile { + readonly description: string; + readonly g: RunCaseIterable; +} + +// Base class for DefaultTestFileLoader and FakeTestFileLoader. +export abstract class TestFileLoader { + abstract listing(suite: string): Promise; + protected abstract import(path: string): Promise; + + importSpecFile(suite: string, path: string[]): Promise { + return this.import(`${suite}/${path.join('/')}.spec.js`); + } + + async loadTree(query: string, subqueriesToExpand: string[] = []): Promise { + return loadTreeForQuery( + this, + parseQuery(query), + subqueriesToExpand.map(q => parseQuery(q)) + ); + } + + async loadTests(query: string): Promise> { + const tree = await this.loadTree(query); + return tree.iterateLeaves(); + } +} + +export class DefaultTestFileLoader extends TestFileLoader { + async listing(suite: string): Promise { + return ((await import(`../../${suite}/listing.js`)) as ListingFile).listing; + } + + import(path: string): Promise { + return import(`../../${path}`); + } +} diff --git a/src/common/framework/fixture.ts b/src/common/framework/fixture.ts index 09d48842ad25..4a73235e1a63 100644 --- a/src/common/framework/fixture.ts +++ b/src/common/framework/fixture.ts @@ -1,5 +1,5 @@ -import { TestCaseRecorder } from './logger.js'; -import { ParamSpec } from './params_utils.js'; +import { TestCaseRecorder } from './logging/test_case_recorder.js'; +import { CaseParams } from './params_utils.js'; import { assert } from './util/util.js'; export class SkipTestCase extends Error {} @@ -13,7 +13,7 @@ export class Fixture { private eventualExpectations: Array> = []; private numOutstandingAsyncExpectations = 0; - constructor(rec: TestCaseRecorder, params: ParamSpec) { + constructor(rec: TestCaseRecorder, params: CaseParams) { this.rec = rec; this.params = params; } @@ -61,16 +61,16 @@ export class Fixture { private expectErrorValue(expectedName: string, ex: unknown, niceStack: Error): void { if (!(ex instanceof Error)) { - niceStack.message = 'THREW non-error value, of type ' + typeof ex + niceStack.message; + niceStack.message = `THREW non-error value, of type ${typeof ex}: ${ex}`; this.rec.fail(niceStack); return; } const actualName = ex.name; if (actualName !== expectedName) { - niceStack.message = `THREW ${actualName}, instead of ${expectedName}` + niceStack.message; + niceStack.message = `THREW ${actualName}, instead of ${expectedName}: ${ex}`; this.rec.fail(niceStack); } else { - niceStack.message = 'OK: threw ' + actualName + niceStack.message; + niceStack.message = `OK: threw ${actualName}${ex.message}`; this.rec.debug(niceStack); } } @@ -80,7 +80,7 @@ export class Fixture { const m = msg ? ': ' + msg : ''; try { await p; - niceStack.message = 'DID NOT THROW' + m; + niceStack.message = 'DID NOT REJECT' + m; this.rec.fail(niceStack); } catch (ex) { niceStack.message = m; diff --git a/src/common/framework/generate_minimal_query_list.ts b/src/common/framework/generate_minimal_query_list.ts deleted file mode 100644 index 32538bbe6211..000000000000 --- a/src/common/framework/generate_minimal_query_list.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { TestSpecOrTestOrCaseID } from './id.js'; -import { Logger } from './logger.js'; -import { makeFilter } from './test_filter/load_filter.js'; -import { TestFilterResult } from './test_filter/test_filter_result.js'; -import { FilterResultTreeNode, treeFromFilterResults } from './tree.js'; - -interface QuerySplitterTreeNode { - needsSplit: boolean; - children?: Map; -} - -interface Expectation { - id: TestSpecOrTestOrCaseID; - line: string; - seen: boolean; -} - -function makeQuerySplitterTree( - caselist: TestFilterResult[], - expectationStrings: string[] -): QuerySplitterTreeNode { - const expectations: Expectation[] = []; - for (const e of expectationStrings) { - const filter = makeFilter(e); - const id = filter.idIfSingle(); - if (!id) { - throw new Error( - 'Can only handle expectations which cover one file, one test, or one case. ' + e - ); - } - expectations.push({ id, line: e, seen: false }); - } - - const convertToQuerySplitterTree = ( - tree: FilterResultTreeNode, - name?: string - ): QuerySplitterTreeNode => { - const children = tree.children; - let needsSplit = true; - - if (name !== undefined) { - const filter = makeFilter(name); - const moreThanOneFile = !filter.definitelyOneFile(); - const matchingExpectations = expectations.map(e => { - const matches = filter.matches(e.id); - if (matches) e.seen = true; - return matches; - }); - needsSplit = matchingExpectations.some(m => m) || moreThanOneFile; - } - - const queryNode: QuerySplitterTreeNode = { needsSplit }; - if (children) { - queryNode.children = new Map(); - for (const [k, v] of children) { - const subtree = convertToQuerySplitterTree(v, k); - queryNode.children.set(k, subtree); - } - } - return queryNode; - }; - - const log = new Logger(); - const tree = treeFromFilterResults(log, caselist.values()); - const queryTree = convertToQuerySplitterTree(tree)!; - - for (const e of expectations) { - if (!e.seen) throw new Error('expectation had no effect: ' + e.line); - } - - return queryTree; -} - -// Takes a TestFilterResultIterator enumerating every test case in the suite, and a list of -// expectation queries from a browser's expectations file. Creates a minimal list of queries -// (i.e. wpt variant lines) such that: -// -// - There is at least one query per spec file. -// - Each of those those input queries is in the output, so that it can have its own expectation. -// -// It does this by creating a tree from the list of cases (same tree as the standalone runner uses), -// then marking every node which is a parent of a node that matches an expectation. -export async function generateMinimalQueryList( - caselist: TestFilterResult[], - expectationStrings: string[] -): Promise { - const unsplitNodes: string[] = []; - const findUnsplitNodes = (name: string, node: QuerySplitterTreeNode | undefined) => { - if (node === undefined) { - return; - } - if (node.needsSplit && node.children) { - for (const [k, v] of node.children) { - findUnsplitNodes(k, v); - } - } else { - unsplitNodes.push(name); - } - }; - - const queryTree = makeQuerySplitterTree(caselist, expectationStrings); - findUnsplitNodes('', queryTree); - - for (const exp of expectationStrings) { - if (!unsplitNodes.some(name => name === exp)) { - throw new Error( - 'Something went wrong: all expectation strings should always appear exactly: ' + exp - ); - } - } - return unsplitNodes; -} diff --git a/src/common/framework/gpu/device_pool.ts b/src/common/framework/gpu/device_pool.ts index 632b717bcf2f..b008aa180c09 100644 --- a/src/common/framework/gpu/device_pool.ts +++ b/src/common/framework/gpu/device_pool.ts @@ -71,7 +71,7 @@ export class DevicePool { // TODO: device.destroy() // Mark the holder as free. (This only has an effect if the pool still has the holder.) - // This could be done at the top but is done here to guard againt async-races during release. + // This could be done at the top but is done here to guard against async-races during release. holder.acquired = false; } } diff --git a/src/common/framework/id.ts b/src/common/framework/id.ts deleted file mode 100644 index 60d04da15986..000000000000 --- a/src/common/framework/id.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { ParamSpec } from './params_utils.js'; - -// Identifies a test spec file. -export interface TestSpecID { - // The spec's suite name, e.g. 'webgpu'. - readonly suite: string; - // The spec's path within the suite, e.g. 'command_buffer/compute/basic'. - readonly path: string; -} - -export function testSpecEquals(x: TestSpecID, y: TestSpecID): boolean { - return x.suite === y.suite && x.path === y.path; -} - -// Identifies a test case (a specific parameterization of a test), within its spec file. -export interface TestCaseID { - readonly test: string; - readonly params: ParamSpec | null; -} - -export interface TestSpecOrTestOrCaseID { - readonly spec: TestSpecID; - readonly test?: string; - readonly params?: ParamSpec | null; -} diff --git a/src/common/framework/listing.ts b/src/common/framework/listing.ts deleted file mode 100644 index bc5bedaf7092..000000000000 --- a/src/common/framework/listing.ts +++ /dev/null @@ -1,9 +0,0 @@ -// A listing of all specs within a single suite. This is the (awaited) type of -// `groups` in '{cts,unittests}/listing.ts' and the auto-generated -// 'out/{cts,unittests}/listing.js' files (see tools/gen_listings). -export type TestSuiteListing = Iterable; - -export interface TestSuiteListingEntry { - readonly path: string; - readonly description: string; -} diff --git a/src/common/framework/loader.ts b/src/common/framework/loader.ts deleted file mode 100644 index 78f804ae2bc3..000000000000 --- a/src/common/framework/loader.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { TestSuiteListing } from './listing.js'; -import { loadFilter } from './test_filter/load_filter.js'; -import { TestFilterResult } from './test_filter/test_filter_result.js'; -import { RunCaseIterable } from './test_group.js'; - -// One of the following: -// - An actual .spec.ts file, as imported. -// - A *filtered* list of cases from a single .spec.ts file. -export interface TestSpec { - readonly description: string; - readonly g: RunCaseIterable; -} - -// A shell object describing a directory (from its README.txt). -export interface ReadmeFile { - readonly description: string; -} - -export type TestSpecOrReadme = TestSpec | ReadmeFile; - -type TestFilterResultIterator = IterableIterator; -function* concat(lists: TestFilterResult[][]): TestFilterResultIterator { - for (const specs of lists) { - yield* specs; - } -} - -export interface TestFileLoader { - listing(suite: string): Promise; - import(path: string): Promise; -} - -class DefaultTestFileLoader implements TestFileLoader { - async listing(suite: string): Promise { - return (await import(`../../${suite}/listing.js`)).listing; - } - - import(path: string): Promise { - return import('../../' + path); - } -} - -export class TestLoader { - private fileLoader: TestFileLoader; - - constructor(fileLoader: TestFileLoader = new DefaultTestFileLoader()) { - this.fileLoader = fileLoader; - } - - // TODO: Test - async loadTestsFromQuery(query: string): Promise { - return this.loadTests(new URLSearchParams(query).getAll('q')); - } - - // TODO: Test - // TODO: Probably should actually not exist at all, just use queries on cmd line too. - async loadTestsFromCmdLine(filters: string[]): Promise { - return this.loadTests(filters); - } - - async loadTests(filters: string[]): Promise { - const loads = filters.map(f => loadFilter(this.fileLoader, f)); - return concat(await Promise.all(loads)); - } -} diff --git a/src/common/framework/logger.ts b/src/common/framework/logger.ts deleted file mode 100644 index 0dc2eb1a1614..000000000000 --- a/src/common/framework/logger.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { SkipTestCase } from './fixture.js'; -import { TestSpecID } from './id.js'; -import { ParamSpec, extractPublicParams } from './params_utils.js'; -import { makeQueryString } from './url_query.js'; -import { getStackTrace } from './util/stack.js'; -import { assert, now } from './util/util.js'; -import { version } from './version.js'; - -type Status = 'running' | 'pass' | 'skip' | 'warn' | 'fail'; -export interface LiveTestSpecResult { - readonly spec: string; - readonly cases: LiveTestCaseResult[]; -} - -interface TestCaseResult { - readonly test: string; - readonly params: ParamSpec | null; - status: Status; - timems: number; -} - -export interface LiveTestCaseResult extends TestCaseResult { - logs?: LogMessageWithStack[]; -} - -export interface TransferredTestCaseResult extends TestCaseResult { - // When transferred from a worker, a LogMessageWithStack turns into a generic Error - // (its prototype gets lost and replaced with Error). - logs?: Error[]; -} - -export class LogMessageWithStack extends Error { - constructor(name: string, ex: Error, includeStack: boolean = true) { - super(ex.message); - - this.name = name; - this.stack = includeStack ? ex.stack : undefined; - } - - toJSON(): string { - let m = this.name; - if (this.message) { - m += ': ' + this.message; - } - if (this.stack) { - m += '\n' + getStackTrace(this); - } - return m; - } -} - -export class Logger { - readonly results: LiveTestSpecResult[] = []; - - constructor() {} - - record(spec: TestSpecID): [TestSpecRecorder, LiveTestSpecResult] { - const result: LiveTestSpecResult = { spec: makeQueryString(spec), cases: [] }; - this.results.push(result); - return [new TestSpecRecorder(result), result]; - } - - asJSON(space?: number): string { - return JSON.stringify({ version, results: this.results }, undefined, space); - } -} - -export class TestSpecRecorder { - private result: LiveTestSpecResult; - - constructor(result: LiveTestSpecResult) { - this.result = result; - } - - record(test: string, params: ParamSpec | null): [TestCaseRecorder, LiveTestCaseResult] { - const result: LiveTestCaseResult = { - test, - params: params ? extractPublicParams(params) : null, - status: 'running', - timems: -1, - }; - this.result.cases.push(result); - return [new TestCaseRecorder(result), result]; - } -} - -enum PassState { - pass = 0, - skip = 1, - warn = 2, - fail = 3, -} - -export class TestCaseRecorder { - private result: LiveTestCaseResult; - private state = PassState.pass; - private startTime = -1; - private logs: LogMessageWithStack[] = []; - private debugging = false; - - constructor(result: LiveTestCaseResult) { - this.result = result; - } - - start(debug: boolean = false): void { - this.startTime = now(); - this.logs = []; - this.state = PassState.pass; - this.debugging = debug; - } - - finish(): void { - assert(this.startTime >= 0, 'finish() before start()'); - - const endTime = now(); - // Round to next microsecond to avoid storing useless .xxxx00000000000002 in results. - this.result.timems = Math.ceil((endTime - this.startTime) * 1000) / 1000; - this.result.status = PassState[this.state] as Status; - - this.result.logs = this.logs; - this.debugging = false; - } - - debug(ex: Error): void { - if (!this.debugging) { - return; - } - this.logs.push(new LogMessageWithStack('DEBUG', ex, false)); - } - - warn(ex: Error): void { - this.setState(PassState.warn); - this.logs.push(new LogMessageWithStack('WARN', ex)); - } - - fail(ex: Error): void { - this.setState(PassState.fail); - this.logs.push(new LogMessageWithStack('FAIL', ex)); - } - - skipped(ex: SkipTestCase): void { - this.setState(PassState.skip); - this.logs.push(new LogMessageWithStack('SKIP', ex)); - } - - threw(ex: Error): void { - if (ex instanceof SkipTestCase) { - this.skipped(ex); - return; - } - - this.setState(PassState.fail); - this.logs.push(new LogMessageWithStack('EXCEPTION', ex)); - } - - private setState(state: PassState): void { - this.state = Math.max(this.state, state); - } -} diff --git a/src/common/framework/logging/log_message.ts b/src/common/framework/logging/log_message.ts new file mode 100644 index 000000000000..525e099b031b --- /dev/null +++ b/src/common/framework/logging/log_message.ts @@ -0,0 +1,21 @@ +import { extractImportantStackTrace } from '../util/stack.js'; + +export class LogMessageWithStack extends Error { + constructor(name: string, ex: Error, includeStack: boolean = true) { + super(ex.message); + + this.name = name; + this.stack = includeStack ? ex.stack : undefined; + } + + toJSON(): string { + let m = this.name + ': '; + if (this.stack) { + // this.message is already included in this.stack + m += extractImportantStackTrace(this); + } else { + m += this.message; + } + return m; + } +} diff --git a/src/common/framework/logging/logger.ts b/src/common/framework/logging/logger.ts new file mode 100644 index 000000000000..4f4bc1c45e5d --- /dev/null +++ b/src/common/framework/logging/logger.ts @@ -0,0 +1,25 @@ +import { version } from '../version.js'; + +import { LiveTestCaseResult } from './result.js'; +import { TestCaseRecorder } from './test_case_recorder.js'; + +export type LogResults = Map; + +export class Logger { + readonly debug: boolean; + readonly results: LogResults = new Map(); + + constructor(debug: boolean) { + this.debug = debug; + } + + record(name: string): [TestCaseRecorder, LiveTestCaseResult] { + const result: LiveTestCaseResult = { status: 'running', timems: -1 }; + this.results.set(name, result); + return [new TestCaseRecorder(result, this.debug), result]; + } + + asJSON(space?: number): string { + return JSON.stringify({ version, results: Array.from(this.results) }, undefined, space); + } +} diff --git a/src/common/framework/logging/result.ts b/src/common/framework/logging/result.ts new file mode 100644 index 000000000000..ebdaf47642b1 --- /dev/null +++ b/src/common/framework/logging/result.ts @@ -0,0 +1,18 @@ +import { LogMessageWithStack } from './log_message.js'; + +export type Status = 'running' | 'pass' | 'skip' | 'warn' | 'fail'; + +export interface TestCaseResult { + status: Status; + timems: number; +} + +export interface LiveTestCaseResult extends TestCaseResult { + logs?: LogMessageWithStack[]; +} + +export interface TransferredTestCaseResult extends TestCaseResult { + // When transferred from a worker, a LogMessageWithStack turns into a generic Error + // (its prototype gets lost and replaced with Error). + logs?: Error[]; +} diff --git a/src/common/framework/logging/test_case_recorder.ts b/src/common/framework/logging/test_case_recorder.ts new file mode 100644 index 000000000000..a6448bc004b1 --- /dev/null +++ b/src/common/framework/logging/test_case_recorder.ts @@ -0,0 +1,82 @@ +import { SkipTestCase } from '../fixture.js'; +import { now, assert } from '../util/util.js'; + +import { LogMessageWithStack } from './log_message.js'; +import { LiveTestCaseResult, Status } from './result.js'; + +enum PassState { + pass = 0, + skip = 1, + warn = 2, + fail = 3, +} + +// Holds onto a LiveTestCaseResult owned by the Logger, and writes the results into it. +export class TestCaseRecorder { + private result: LiveTestCaseResult; + private state = PassState.pass; + private startTime = -1; + private logs: LogMessageWithStack[] = []; + private debugging = false; + + constructor(result: LiveTestCaseResult, debugging: boolean) { + this.result = result; + this.debugging = debugging; + } + + start(): void { + assert(this.startTime < 0, 'TestCaseRecorder cannot be reused'); + this.startTime = now(); + } + + finish(): void { + assert(this.startTime >= 0, 'finish() before start()'); + + const timeMilliseconds = now() - this.startTime; + // Round to next microsecond to avoid storing useless .xxxx00000000000002 in results. + this.result.timems = Math.ceil(timeMilliseconds * 1000) / 1000; + this.result.status = PassState[this.state] as Status; // Convert numeric enum back to string + + this.result.logs = this.logs; + } + + injectResult(injectedResult: LiveTestCaseResult): void { + Object.assign(this.result, injectedResult); + } + + debug(ex: Error): void { + if (!this.debugging) { + return; + } + this.logs.push(new LogMessageWithStack('DEBUG', ex, false)); + } + + warn(ex: Error): void { + this.setState(PassState.warn); + this.logs.push(new LogMessageWithStack('WARN', ex)); + } + + fail(ex: Error): void { + this.setState(PassState.fail); + this.logs.push(new LogMessageWithStack('FAIL', ex)); + } + + skipped(ex: SkipTestCase): void { + this.setState(PassState.skip); + this.logs.push(new LogMessageWithStack('SKIP', ex)); + } + + threw(ex: Error): void { + if (ex instanceof SkipTestCase) { + this.skipped(ex); + return; + } + + this.setState(PassState.fail); + this.logs.push(new LogMessageWithStack('EXCEPTION', ex)); + } + + private setState(state: PassState): void { + this.state = Math.max(this.state, state); + } +} diff --git a/src/common/framework/params.ts b/src/common/framework/params_builder.ts similarity index 88% rename from src/common/framework/params.ts rename to src/common/framework/params_builder.ts index 2e36cf2aed9c..9b18c9f30d6b 100644 --- a/src/common/framework/params.ts +++ b/src/common/framework/params_builder.ts @@ -1,4 +1,4 @@ -import { ParamSpec, ParamSpecIterable, paramsEquals } from './params_utils.js'; +import { CaseParams, CaseParamsIterable, publicParamsEquals } from './params_utils.js'; import { assert } from './util/util.js'; // https://stackoverflow.com/a/56375136 @@ -33,11 +33,11 @@ export function params(): ParamsBuilder<{}> { return new ParamsBuilder(); } -class ParamsBuilder implements ParamSpecIterable { - private paramSpecs: ParamSpecIterable = [{}]; +export class ParamsBuilder implements CaseParamsIterable { + private paramSpecs: CaseParamsIterable = [{}]; [Symbol.iterator](): Iterator { - const iter: Iterator = this.paramSpecs[Symbol.iterator](); + const iter: Iterator = this.paramSpecs[Symbol.iterator](); return iter as Iterator; } @@ -49,7 +49,7 @@ class ParamsBuilder implements ParamSpecIterable { yield mergeParams(a, b); } } - }); + }) as CaseParamsIterable; /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ return this as any; } @@ -62,7 +62,7 @@ class ParamsBuilder implements ParamSpecIterable { yield mergeParams(a, b); } } - }); + }) as CaseParamsIterable; /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ return this as any; } @@ -83,12 +83,12 @@ class ParamsBuilder implements ParamSpecIterable { return this.filter(x => !pred(x)); } - exclude(exclude: ParamSpecIterable): ParamsBuilder { + exclude(exclude: CaseParamsIterable): ParamsBuilder { const excludeArray = Array.from(exclude); const paramSpecs = this.paramSpecs; this.paramSpecs = makeReusableIterable(function* () { for (const p of paramSpecs) { - if (excludeArray.every(e => !paramsEquals(p, e))) { + if (excludeArray.every(e => !publicParamsEquals(p, e))) { yield p; } } diff --git a/src/common/framework/params_utils.ts b/src/common/framework/params_utils.ts index 2f80e42f1108..b16f21870d11 100644 --- a/src/common/framework/params_utils.ts +++ b/src/common/framework/params_utils.ts @@ -1,69 +1,32 @@ -import { objectEquals } from './util/util.js'; +import { comparePublicParamsPaths, Ordering } from './query/compare.js'; +import { kWildcard, kParamSeparator } from './query/separators.js'; -/* eslint-disable-next-line @typescript-eslint/no-explicit-any */ -export type ParamArgument = any; -export interface ParamSpec { +// Consider adding more types here if needed +export type ParamArgument = void | undefined | number | string | boolean | number[]; +export interface CaseParams { + readonly [k: string]: ParamArgument; +} +export interface CaseParamsRW { [k: string]: ParamArgument; } -export type ParamSpecIterable = Iterable; -export type ParamSpecIterator = IterableIterator; +export type CaseParamsIterable = Iterable; + +export function paramKeyIsPublic(key: string): boolean { + return !key.startsWith('_'); +} -export function extractPublicParams(params: ParamSpec): ParamSpec { - const publicParams: ParamSpec = {}; +export function extractPublicParams(params: CaseParams): CaseParams { + const publicParams: CaseParamsRW = {}; for (const k of Object.keys(params)) { - if (!k.startsWith('_')) { + if (paramKeyIsPublic(k)) { publicParams[k] = params[k]; } } return publicParams; } -export function stringifyPublicParams(p: ParamSpec | null): string { - if (p === null || paramsEquals(p, {})) { - return ''; - } - return JSON.stringify(extractPublicParams(p)); -} +export const badParamValueChars = new RegExp('[=' + kParamSeparator + kWildcard + ']'); -export function paramsEquals(x: ParamSpec | null, y: ParamSpec | null): boolean { - if (x === y) { - return true; - } - if (x === null) { - x = {}; - } - if (y === null) { - y = {}; - } - - for (const xk of Object.keys(x)) { - if (x[xk] !== undefined && !(xk in y)) { - return false; - } - if (!objectEquals(x[xk], y[xk])) { - return false; - } - } - - for (const yk of Object.keys(y)) { - if (y[yk] !== undefined && !(yk in x)) { - return false; - } - } - return true; -} - -export function paramsSupersets(sup: ParamSpec | null, sub: ParamSpec | null): boolean { - if (sub === null) { - return true; - } - if (sup === null) { - sup = {}; - } - for (const k of Object.keys(sub)) { - if (!(k in sup) || sup[k] !== sub[k]) { - return false; - } - } - return true; +export function publicParamsEquals(x: CaseParams, y: CaseParams): boolean { + return comparePublicParamsPaths(x, y) === Ordering.Equal; } diff --git a/src/common/framework/query/compare.ts b/src/common/framework/query/compare.ts new file mode 100644 index 000000000000..9d3ff2a545a3 --- /dev/null +++ b/src/common/framework/query/compare.ts @@ -0,0 +1,89 @@ +import { CaseParams, extractPublicParams } from '../params_utils.js'; +import { assert, objectEquals } from '../util/util.js'; + +import { TestQuery } from './query.js'; + +export const enum Ordering { + Unordered, + StrictSuperset, + Equal, + StrictSubset, +} + +// Compares two queries for their ordering (which is used to build the tree). +// See src/unittests/query_compare.spec.ts for examples. +export function compareQueries(a: TestQuery, b: TestQuery): Ordering { + if (a.suite !== b.suite) { + return Ordering.Unordered; + } + + const filePathOrdering = comparePaths(a.filePathParts, b.filePathParts); + if (filePathOrdering !== Ordering.Equal || a.isMultiFile || b.isMultiFile) { + return compareOneLevel(filePathOrdering, a.isMultiFile, b.isMultiFile); + } + assert('testPathParts' in a && 'testPathParts' in b); + + const testPathOrdering = comparePaths(a.testPathParts, b.testPathParts); + if (testPathOrdering !== Ordering.Equal || a.isMultiTest || b.isMultiTest) { + return compareOneLevel(testPathOrdering, a.isMultiTest, b.isMultiTest); + } + assert('params' in a && 'params' in b); + + const paramsPathOrdering = comparePublicParamsPaths(a.params, b.params); + if (paramsPathOrdering !== Ordering.Equal || a.isMultiCase || b.isMultiCase) { + return compareOneLevel(paramsPathOrdering, a.isMultiCase, b.isMultiCase); + } + return Ordering.Equal; +} + +// Compares a single level of a query. +// "IsBig" means the query is big relative to the level, e.g. for test-level: +// anything >= suite:a,* is big +// anything <= suite:a:* is small +function compareOneLevel(ordering: Ordering, aIsBig: boolean, bIsBig: boolean): Ordering { + assert(ordering !== Ordering.Equal || aIsBig || bIsBig); + if (ordering === Ordering.Unordered) return Ordering.Unordered; + if (aIsBig && bIsBig) return ordering; + if (!aIsBig && !bIsBig) return Ordering.Unordered; // Equal case is already handled + // Exactly one of (a, b) is big. + if (aIsBig && ordering !== Ordering.StrictSubset) return Ordering.StrictSuperset; + if (bIsBig && ordering !== Ordering.StrictSuperset) return Ordering.StrictSubset; + return Ordering.Unordered; +} + +function comparePaths(a: readonly string[], b: readonly string[]): Ordering { + const shorter = Math.min(a.length, b.length); + + for (let i = 0; i < shorter; ++i) { + if (a[i] !== b[i]) { + return Ordering.Unordered; + } + } + if (a.length === b.length) { + return Ordering.Equal; + } else if (a.length < b.length) { + return Ordering.StrictSuperset; + } else { + return Ordering.StrictSubset; + } +} + +export function comparePublicParamsPaths(a0: CaseParams, b0: CaseParams): Ordering { + const a = extractPublicParams(a0); + const b = extractPublicParams(b0); + const aKeys = Object.keys(a); + const commonKeys = new Set(aKeys.filter(k => k in b)); + + for (const k of commonKeys) { + if (!objectEquals(a[k], b[k])) { + return Ordering.Unordered; + } + } + const bKeys = Object.keys(b); + const aRemainingKeys = aKeys.length - commonKeys.size; + const bRemainingKeys = bKeys.length - commonKeys.size; + if (aRemainingKeys === 0 && bRemainingKeys === 0) return Ordering.Equal; + if (aRemainingKeys === 0) return Ordering.StrictSuperset; + if (bRemainingKeys === 0) return Ordering.StrictSubset; + return Ordering.Unordered; +} diff --git a/src/common/framework/query/encode_selectively.ts b/src/common/framework/query/encode_selectively.ts new file mode 100644 index 000000000000..883dc26c6a2a --- /dev/null +++ b/src/common/framework/query/encode_selectively.ts @@ -0,0 +1,16 @@ +// Encodes a stringified TestQuery so that it can be placed in a `?q=` parameter in a URL. +// encodeURIComponent encodes in accordance with `application/x-www-form-urlencoded`, but URLs don't +// actually have to be as strict as HTML form encoding (we interpret this purely from JavaScript). +// So we encode the component, then selectively convert some %-encoded escape codes back to their +// original form for readability/copyability. +export function encodeURIComponentSelectively(s: string): string { + let ret = encodeURIComponent(s); + ret = ret.replace(/%22/g, '"'); // for JSON strings + ret = ret.replace(/%2C/g, ','); // for path separator, and JSON arrays + ret = ret.replace(/%3A/g, ':'); // for big separator + ret = ret.replace(/%3B/g, ';'); // for param separator + ret = ret.replace(/%3D/g, '='); // for params (k=v) + ret = ret.replace(/%5B/g, '['); // for JSON arrays + ret = ret.replace(/%5D/g, ']'); // for JSON arrays + return ret; +} diff --git a/src/common/framework/query/parseQuery.ts b/src/common/framework/query/parseQuery.ts new file mode 100644 index 000000000000..029ad4fc4fc0 --- /dev/null +++ b/src/common/framework/query/parseQuery.ts @@ -0,0 +1,132 @@ +import { + CaseParamsRW, + ParamArgument, + badParamValueChars, + paramKeyIsPublic, +} from '../params_utils.js'; +import { assert } from '../util/util.js'; + +import { + TestQuery, + TestQueryMultiFile, + TestQueryMultiTest, + TestQueryMultiCase, + TestQuerySingleCase, +} from './query.js'; +import { kBigSeparator, kWildcard, kPathSeparator, kParamSeparator } from './separators.js'; +import { validQueryPart } from './validQueryPart.js'; + +export function parseQuery(s: string): TestQuery { + try { + return parseQueryImpl(s); + } catch (ex) { + ex.message += '\n on: ' + s; + throw ex; + } +} + +function parseQueryImpl(s: string): TestQuery { + // Undo encodeURIComponentSelectively + s = decodeURIComponent(s); + + // bigParts are: suite, group, test, params (note kBigSeparator could appear in params) + const [suite, fileString, testString, paramsString] = s.split(kBigSeparator, 4); + assert(fileString !== undefined, `filter string must have at least one ${kBigSeparator}`); + + const { parts: file, wildcard: filePathHasWildcard } = parseBigPart(fileString, kPathSeparator); + + if (testString === undefined) { + // Query is file-level + assert( + filePathHasWildcard, + `File-level query without wildcard ${kWildcard}. Did you want a file-level query \ +(append ${kPathSeparator}${kWildcard}) or test-level query (append ${kBigSeparator}${kWildcard})?` + ); + return new TestQueryMultiFile(suite, file); + } + assert(!filePathHasWildcard, `Wildcard ${kWildcard} must be at the end of the query string`); + + const { parts: test, wildcard: testPathHasWildcard } = parseBigPart(testString, kPathSeparator); + + if (paramsString === undefined) { + // Query is test-level + assert( + testPathHasWildcard, + `Test-level query without wildcard ${kWildcard}; did you want a test-level query \ +(append ${kPathSeparator}${kWildcard}) or case-level query (append ${kBigSeparator}${kWildcard})?` + ); + assert(file.length > 0, 'File part of test-level query was empty (::)'); + return new TestQueryMultiTest(suite, file, test); + } + + // Query is case-level + assert(!testPathHasWildcard, `Wildcard ${kWildcard} must be at the end of the query string`); + + const { parts: paramsParts, wildcard: paramsHasWildcard } = parseBigPart( + paramsString, + kParamSeparator + ); + + assert(test.length > 0, 'Test part of case-level query was empty (::)'); + + const params: CaseParamsRW = {}; + for (const paramPart of paramsParts) { + const [k, v] = parseSingleParam(paramPart); + assert(validQueryPart.test(k), 'param key names must match ' + validQueryPart); + params[k] = v; + } + if (paramsHasWildcard) { + return new TestQueryMultiCase(suite, file, test, params); + } else { + return new TestQuerySingleCase(suite, file, test, params); + } +} + +// webgpu:a,b,* or webgpu:a,b,c:* +const kExampleQueries = `\ +webgpu${kBigSeparator}a${kPathSeparator}b${kPathSeparator}${kWildcard} or \ +webgpu${kBigSeparator}a${kPathSeparator}b${kPathSeparator}c${kBigSeparator}${kWildcard}`; + +function parseBigPart( + s: string, + separator: typeof kParamSeparator | typeof kPathSeparator +): { parts: string[]; wildcard: boolean } { + if (s === '') { + return { parts: [], wildcard: false }; + } + const parts = s.split(separator); + + let endsWithWildcard = false; + for (const [i, part] of parts.entries()) { + if (i === parts.length - 1) { + endsWithWildcard = part === kWildcard; + } + assert( + part.indexOf(kWildcard) === -1 || endsWithWildcard, + `Wildcard ${kWildcard} must be complete last part of a path (e.g. ${kExampleQueries})` + ); + } + if (endsWithWildcard) { + // Remove the last element of the array (which is just the wildcard). + parts.length = parts.length - 1; + } + return { parts, wildcard: endsWithWildcard }; +} + +function parseSingleParam(paramSubstring: string): [string, ParamArgument] { + assert(paramSubstring !== '', 'Param in a query must not be blank (is there a trailing comma?)'); + const i = paramSubstring.indexOf('='); + assert(i !== -1, 'Param in a query must be of form key=value'); + const k = paramSubstring.substring(0, i); + assert(paramKeyIsPublic(k), 'Param in a query must not be private (start with _)'); + const v = paramSubstring.substring(i + 1); + return [k, parseSingleParamValue(v)]; +} + +function parseSingleParamValue(s: string): ParamArgument { + assert( + !badParamValueChars.test(s), + `param value must not match ${badParamValueChars} - was ${s}` + ); + return s === 'undefined' ? undefined : JSON.parse(s); +} diff --git a/src/common/framework/query/query.ts b/src/common/framework/query/query.ts new file mode 100644 index 000000000000..665fef66143c --- /dev/null +++ b/src/common/framework/query/query.ts @@ -0,0 +1,91 @@ +import { CaseParams } from '../params_utils.js'; +import { assert } from '../util/util.js'; + +import { encodeURIComponentSelectively } from './encode_selectively.js'; +import { kBigSeparator, kPathSeparator, kWildcard, kParamSeparator } from './separators.js'; +import { stringifyPublicParams } from './stringify_params.js'; + +export type TestQuery = + | TestQuerySingleCase + | TestQueryMultiCase + | TestQueryMultiTest + | TestQueryMultiFile; + +export class TestQueryMultiFile { + readonly isMultiFile: boolean = true; + readonly suite: string; + readonly filePathParts: readonly string[]; + + constructor(suite: string, file: readonly string[]) { + this.suite = suite; + this.filePathParts = [...file]; + } + + toString(): string { + return encodeURIComponentSelectively(this.toStringHelper().join(kBigSeparator)); + } + + toHTML(): string { + return this.toStringHelper().join(kBigSeparator + ''); + } + + protected toStringHelper(): string[] { + return [this.suite, [...this.filePathParts, kWildcard].join(kPathSeparator)]; + } +} + +export class TestQueryMultiTest extends TestQueryMultiFile { + readonly isMultiFile: false = false; + readonly isMultiTest: boolean = true; + readonly testPathParts: readonly string[]; + + constructor(suite: string, file: readonly string[], test: readonly string[]) { + super(suite, file); + assert(file.length > 0, 'multi-test (or finer) query must have file-path'); + this.testPathParts = [...test]; + } + + protected toStringHelper(): string[] { + return [ + this.suite, + this.filePathParts.join(kPathSeparator), + [...this.testPathParts, kWildcard].join(kPathSeparator), + ]; + } +} + +export class TestQueryMultiCase extends TestQueryMultiTest { + readonly isMultiTest: false = false; + readonly isMultiCase: boolean = true; + readonly params: CaseParams; + + constructor(suite: string, file: readonly string[], test: readonly string[], params: CaseParams) { + super(suite, file, test); + assert(test.length > 0, 'multi-case (or finer) query must have test-path'); + this.params = { ...params }; + } + + protected toStringHelper(): string[] { + const paramsParts = stringifyPublicParams(this.params); + return [ + this.suite, + this.filePathParts.join(kPathSeparator), + this.testPathParts.join(kPathSeparator), + [...paramsParts, kWildcard].join(kParamSeparator), + ]; + } +} + +export class TestQuerySingleCase extends TestQueryMultiCase { + readonly isMultiCase: false = false; + + protected toStringHelper(): string[] { + const paramsParts = stringifyPublicParams(this.params); + return [ + this.suite, + this.filePathParts.join(kPathSeparator), + this.testPathParts.join(kPathSeparator), + paramsParts.join(kParamSeparator), + ]; + } +} diff --git a/src/common/framework/query/separators.ts b/src/common/framework/query/separators.ts new file mode 100644 index 000000000000..f6e81deb0001 --- /dev/null +++ b/src/common/framework/query/separators.ts @@ -0,0 +1,5 @@ +export const kBigSeparator = ':'; // Separator between big parts: suite:file:test:case +export const kPathSeparator = ','; // Separator between path,to,file or path,to,test +export const kParamSeparator = ';'; // Separator between k=v;k=v +export const kParamKVSeparator = '='; // Separator between key and value in k=v +export const kWildcard = '*'; // Final wildcard, if query is not single-case diff --git a/src/common/framework/query/stringify_params.ts b/src/common/framework/query/stringify_params.ts new file mode 100644 index 000000000000..57bec1a4bbe3 --- /dev/null +++ b/src/common/framework/query/stringify_params.ts @@ -0,0 +1,27 @@ +import { + CaseParams, + extractPublicParams, + ParamArgument, + badParamValueChars, +} from '../params_utils.js'; +import { assert } from '../util/util.js'; + +import { kParamKVSeparator } from './separators.js'; + +export function stringifyPublicParams(p: CaseParams): string[] { + const pub = extractPublicParams(p); + return Object.entries(pub).map(([k, v]) => stringifySingleParam(k, v)); +} + +export function stringifySingleParam(k: string, v: ParamArgument) { + return `${k}${kParamKVSeparator}${stringifySingleParamValue(v)}`; +} + +function stringifySingleParamValue(v: ParamArgument): string { + const s = v === undefined ? 'undefined' : JSON.stringify(v); + assert( + !badParamValueChars.test(s), + `JSON.stringified param value must not match ${badParamValueChars} - was ${s}` + ); + return s; +} diff --git a/src/common/framework/query/validQueryPart.ts b/src/common/framework/query/validQueryPart.ts new file mode 100644 index 000000000000..7693ae9db943 --- /dev/null +++ b/src/common/framework/query/validQueryPart.ts @@ -0,0 +1,2 @@ +// Applies to group parts, test parts, params keys. +export const validQueryPart = /^[a-zA-Z0-9_]+$/; diff --git a/src/common/framework/test_filter/filter_by_group.ts b/src/common/framework/test_filter/filter_by_group.ts deleted file mode 100644 index 021c4078ae2d..000000000000 --- a/src/common/framework/test_filter/filter_by_group.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { TestSpecOrTestOrCaseID } from '../id.js'; -import { ReadmeFile, TestFileLoader, TestSpec } from '../loader.js'; - -import { TestFilter } from './internal.js'; -import { TestFilterResult } from './test_filter_result.js'; - -export class FilterByGroup implements TestFilter { - private readonly suite: string; - private readonly specPathPrefix: string; - - constructor(suite: string, groupPrefix: string) { - this.suite = suite; - this.specPathPrefix = groupPrefix; - } - - matches(id: TestSpecOrTestOrCaseID): boolean { - return id.spec.suite === this.suite && this.pathMatches(id.spec.path); - } - - async iterate(loader: TestFileLoader): Promise { - const specs = await loader.listing(this.suite); - const entries: TestFilterResult[] = []; - - const suite = this.suite; - for (const { path, description } of specs) { - if (this.pathMatches(path)) { - const isReadme = path === '' || path.endsWith('/'); - const spec = isReadme - ? ({ description } as ReadmeFile) - : ((await loader.import(`${suite}/${path}.spec.js`)) as TestSpec); - entries.push({ id: { suite, path }, spec }); - } - } - - return entries; - } - - definitelyOneFile(): boolean { - // FilterByGroup could always possibly match multiple files, because it represents a prefix, - // e.g. "a:b" not "a:b:". - return false; - } - - idIfSingle(): undefined { - // FilterByGroup could be one whole suite, but we only want whole files, tests, or cases. - return undefined; - } - - private pathMatches(path: string): boolean { - return path.startsWith(this.specPathPrefix); - } -} diff --git a/src/common/framework/test_filter/filter_one_file.ts b/src/common/framework/test_filter/filter_one_file.ts deleted file mode 100644 index c6cc99694589..000000000000 --- a/src/common/framework/test_filter/filter_one_file.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { TestCaseID, TestSpecID, TestSpecOrTestOrCaseID, testSpecEquals } from '../id.js'; -import { TestFileLoader, TestSpec } from '../loader.js'; -import { TestSpecRecorder } from '../logger.js'; -import { ParamSpec, paramsEquals, paramsSupersets } from '../params_utils.js'; -import { RunCase, RunCaseIterable } from '../test_group.js'; - -import { TestFilter } from './internal.js'; -import { TestFilterResult } from './test_filter_result.js'; - -abstract class FilterOneFile implements TestFilter { - protected readonly specId: TestSpecID; - - constructor(specId: TestSpecID) { - this.specId = specId; - } - - abstract getCases(spec: TestSpec): RunCaseIterable; - - async iterate(loader: TestFileLoader): Promise { - const spec = (await loader.import( - `${this.specId.suite}/${this.specId.path}.spec.js` - )) as TestSpec; - return [ - { - id: this.specId, - spec: { - description: spec.description, - g: this.getCases(spec), - }, - }, - ]; - } - - definitelyOneFile(): true { - return true; - } - - abstract idIfSingle(): TestSpecOrTestOrCaseID | undefined; - abstract matches(id: TestSpecOrTestOrCaseID): boolean; -} - -type TestGroupFilter = (testcase: TestCaseID) => boolean; -function filterTestGroup(group: RunCaseIterable, filter: TestGroupFilter): RunCaseIterable { - return { - *iterate(log: TestSpecRecorder): Iterable { - for (const rc of group.iterate(log)) { - if (filter(rc.id)) { - yield rc; - } - } - }, - }; -} - -export class FilterByTestMatch extends FilterOneFile { - private readonly testPrefix: string; - - constructor(specId: TestSpecID, testPrefix: string) { - super(specId); - this.testPrefix = testPrefix; - } - - getCases(spec: TestSpec): RunCaseIterable { - return filterTestGroup(spec.g, testcase => this.testMatches(testcase.test)); - } - - idIfSingle(): TestSpecOrTestOrCaseID | undefined { - if (this.testPrefix.length !== 0) { - return undefined; - } - // This is one whole spec file. - return { spec: this.specId }; - } - - matches(id: TestSpecOrTestOrCaseID): boolean { - if (id.test === undefined) { - return false; - } - return testSpecEquals(id.spec, this.specId) && this.testMatches(id.test); - } - - private testMatches(test: string): boolean { - return test.startsWith(this.testPrefix); - } -} - -export class FilterByParamsMatch extends FilterOneFile { - private readonly test: string; - private readonly params: ParamSpec | null; - - constructor(specId: TestSpecID, test: string, params: ParamSpec | null) { - super(specId); - this.test = test; - this.params = params; - } - - getCases(spec: TestSpec): RunCaseIterable { - return filterTestGroup(spec.g, testcase => this.caseMatches(testcase.test, testcase.params)); - } - - idIfSingle(): TestSpecOrTestOrCaseID | undefined { - if (this.params !== null) { - return undefined; - } - // This is one whole test. - return { spec: this.specId, test: this.test }; - } - - matches(id: TestSpecOrTestOrCaseID): boolean { - if (id.test === undefined) { - return false; - } - return testSpecEquals(id.spec, this.specId) && this.caseMatches(id.test, id.params); - } - - private caseMatches(test: string, params: ParamSpec | null | undefined): boolean { - if (params === undefined) { - return false; - } - return test === this.test && paramsSupersets(params, this.params); - } -} - -export class FilterByParamsExact extends FilterOneFile { - private readonly test: string; - private readonly params: ParamSpec | null; - - constructor(specId: TestSpecID, test: string, params: ParamSpec | null) { - super(specId); - this.test = test; - this.params = params; - } - - getCases(spec: TestSpec): RunCaseIterable { - return filterTestGroup(spec.g, testcase => this.caseMatches(testcase.test, testcase.params)); - } - - idIfSingle(): TestSpecOrTestOrCaseID | undefined { - // This is one single test case. - return { spec: this.specId, test: this.test, params: this.params }; - } - - matches(id: TestSpecOrTestOrCaseID): boolean { - if (id.test === undefined || id.params === undefined) { - return false; - } - return testSpecEquals(id.spec, this.specId) && this.caseMatches(id.test, id.params); - } - - private caseMatches(test: string, params: ParamSpec | null): boolean { - return test === this.test && paramsEquals(params, this.params); - } -} diff --git a/src/common/framework/test_filter/internal.ts b/src/common/framework/test_filter/internal.ts deleted file mode 100644 index 5f22b4d763af..000000000000 --- a/src/common/framework/test_filter/internal.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { TestSpecOrTestOrCaseID } from '../id.js'; -import { TestFileLoader } from '../loader.js'; - -import { TestFilterResult } from './test_filter_result.js'; - -export interface TestFilter { - // Iterates over the test cases matched by the filter. - iterate(loader: TestFileLoader): Promise; - - // Iff the filter could not possibly match multiple files, returns true. - definitelyOneFile(): boolean; - - // If the filter can accept one spec, one test, or one case, returns its ID. - idIfSingle(): TestSpecOrTestOrCaseID | undefined; - - matches(id: TestSpecOrTestOrCaseID): boolean; -} diff --git a/src/common/framework/test_filter/load_filter.ts b/src/common/framework/test_filter/load_filter.ts deleted file mode 100644 index 08096ffd056e..000000000000 --- a/src/common/framework/test_filter/load_filter.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { allowedTestNameCharacters } from '../allowed_characters.js'; -import { TestFileLoader } from '../loader.js'; -import { ParamSpec } from '../params_utils.js'; -import { assert, unreachable } from '../util/util.js'; - -import { FilterByGroup } from './filter_by_group.js'; -import { FilterByParamsExact, FilterByParamsMatch, FilterByTestMatch } from './filter_one_file.js'; -import { TestFilter } from './internal.js'; -import { TestFilterResult } from './test_filter_result.js'; - -// Each filter is of one of the forms below (urlencoded). -export function makeFilter(filter: string): TestFilter { - const i1 = filter.indexOf(':'); - assert(i1 !== -1, 'Test queries must fully specify their suite name (e.g. "webgpu:")'); - - const suite = filter.substring(0, i1); - const i2 = filter.indexOf(':', i1 + 1); - if (i2 === -1) { - // - webgpu: - // - webgpu:buf - // - webgpu:buffers/ - // - webgpu:buffers/map - const groupPrefix = filter.substring(i1 + 1); - return new FilterByGroup(suite, groupPrefix); - } - - const path = filter.substring(i1 + 1, i2); - const endOfTestName = new RegExp('[^' + allowedTestNameCharacters + ']'); - const i3sub = filter.substring(i2 + 1).search(endOfTestName); - if (i3sub === -1) { - // - webgpu:buffers/mapWriteAsync: - // - webgpu:buffers/mapWriteAsync:b - const testPrefix = filter.substring(i2 + 1); - return new FilterByTestMatch({ suite, path }, testPrefix); - } - - const i3 = i2 + 1 + i3sub; - const test = filter.substring(i2 + 1, i3); - const token = filter.charAt(i3); - - let params = null; - if (i3 + 1 < filter.length) { - params = JSON.parse(filter.substring(i3 + 1)) as ParamSpec; - } - - if (token === '~') { - // - webgpu:buffers/mapWriteAsync:basic~ - // - webgpu:buffers/mapWriteAsync:basic~{} - // - webgpu:buffers/mapWriteAsync:basic~{filter:"params"} - return new FilterByParamsMatch({ suite, path }, test, params); - } else if (token === '=') { - // - webgpu:buffers/mapWriteAsync:basic= - // - webgpu:buffers/mapWriteAsync:basic={} - // - webgpu:buffers/mapWriteAsync:basic={exact:"params"} - return new FilterByParamsExact({ suite, path }, test, params); - } else { - unreachable("invalid character after test name; must be '~' or '='"); - } -} - -export function loadFilter(loader: TestFileLoader, filter: string): Promise { - return makeFilter(filter).iterate(loader); -} diff --git a/src/common/framework/test_filter/test_filter_result.ts b/src/common/framework/test_filter/test_filter_result.ts deleted file mode 100644 index 934d94eb3316..000000000000 --- a/src/common/framework/test_filter/test_filter_result.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { TestSpecID } from '../id.js'; -import { TestSpecOrReadme } from '../loader.js'; - -// Result of iterating a test filter. Contains a loaded spec (.spec.ts) file and its id. -export interface TestFilterResult { - readonly id: TestSpecID; - readonly spec: TestSpecOrReadme; -} diff --git a/src/common/framework/test_group.ts b/src/common/framework/test_group.ts index ae2589019f82..7a745829cb54 100644 --- a/src/common/framework/test_group.ts +++ b/src/common/framework/test_group.ts @@ -1,27 +1,50 @@ -import { allowedTestNameCharacters } from './allowed_characters.js'; import { Fixture } from './fixture.js'; -import { TestCaseID } from './id.js'; -import { LiveTestCaseResult, TestCaseRecorder, TestSpecRecorder } from './logger.js'; -import { ParamSpec, ParamSpecIterable, extractPublicParams, paramsEquals } from './params_utils.js'; -import { checkPublicParamType } from './url_query.js'; +import { TestCaseRecorder } from './logging/test_case_recorder.js'; +import { + CaseParams, + CaseParamsIterable, + extractPublicParams, + publicParamsEquals, +} from './params_utils.js'; +import { kPathSeparator } from './query/separators.js'; +import { stringifySingleParam } from './query/stringify_params.js'; +import { validQueryPart } from './query/validQueryPart.js'; import { assert } from './util/util.js'; +export type RunFn = (rec: TestCaseRecorder) => Promise; + +export interface TestCaseID { + readonly test: readonly string[]; + readonly params: CaseParams; +} + export interface RunCase { readonly id: TestCaseID; - run(debug?: boolean): Promise; - injectResult(result: LiveTestCaseResult): void; + run: RunFn; +} + +// Interface for defining tests +export interface TestGroupBuilder { + test(name: string): TestBuilderWithName; +} +export function makeTestGroup(fixture: FixtureClass): TestGroupBuilder { + return new TestGroup(fixture); } +// Interface for running tests export interface RunCaseIterable { - iterate(rec: TestSpecRecorder): Iterable; + iterate(): Iterable; +} +export function makeTestGroupForUnitTesting( + fixture: FixtureClass +): TestGroup { + return new TestGroup(fixture); } -type FixtureClass = new (log: TestCaseRecorder, params: ParamSpec) => F; +type FixtureClass = new (log: TestCaseRecorder, params: CaseParams) => F; type TestFn = (t: F & { params: P }) => Promise | void; -const validNames = new RegExp('^[' + allowedTestNameCharacters + ']+$'); - -export class TestGroup implements RunCaseIterable { +class TestGroup implements RunCaseIterable, TestGroupBuilder { private fixture: FixtureClass; private seen: Set = new Set(); private tests: Array> = []; @@ -30,14 +53,13 @@ export class TestGroup implements RunCaseIterable { this.fixture = fixture; } - *iterate(log: TestSpecRecorder): Iterable { + *iterate(): Iterable { for (const test of this.tests) { - yield* test.iterate(log); + yield* test.iterate(); } } private checkName(name: string): void { - assert(validNames.test(name), `Invalid test name ${name}; must match [${validNames}]+`); assert( // Shouldn't happen due to the rule above. Just makes sure that treated // unencoded strings as encoded strings is OK. @@ -51,13 +73,14 @@ export class TestGroup implements RunCaseIterable { // TODO: This could take a fixture, too, to override the one for the group. test(name: string): TestBuilderWithName { - // Replace spaces with underscores for readability. - assert(name.indexOf('_') === -1, 'Invalid test name ${name}: contains underscore (use space)'); - name = name.replace(/ /g, '_'); - this.checkName(name); - const test = new TestBuilder(name, this.fixture); + const parts = name.split(kPathSeparator); + for (const p of parts) { + assert(validQueryPart.test(p), `Invalid test name part ${p}; must match ${validQueryPart}`); + } + + const test = new TestBuilder(parts, this.fixture); this.tests.push(test); return test; } @@ -72,13 +95,13 @@ interface TestBuilderWithParams { } class TestBuilder { - private readonly name: string; + private readonly testPath: string[]; private readonly fixture: FixtureClass; private testFn: TestFn | undefined; - private cases: ParamSpecIterable | null = null; + private cases?: CaseParamsIterable = undefined; - constructor(name: string, fixture: FixtureClass) { - this.name = name; + constructor(testPath: string[], fixture: FixtureClass) { + this.testPath = testPath; this.fixture = fixture; } @@ -86,22 +109,22 @@ class TestBuilder { this.testFn = fn; } - params(specs: Iterable): TestBuilderWithParams { - assert(this.cases === null, 'test case is already parameterized'); - const cases = Array.from(specs); - const seen: ParamSpec[] = []; + params(casesIterable: Iterable): TestBuilderWithParams { + assert(this.cases === undefined, 'test case is already parameterized'); + const cases = Array.from(casesIterable); + const seen: CaseParams[] = []; // This is n^2. for (const spec of cases) { const publicParams = extractPublicParams(spec); // Check type of public params: can only be (currently): // number, string, boolean, undefined, number[] - for (const v of Object.values(publicParams)) { - checkPublicParamType(v); + for (const [k, v] of Object.entries(publicParams)) { + stringifySingleParam(k, v); // To check for invalid params values } assert( - !seen.some(x => paramsEquals(x, publicParams)), + !seen.some(x => publicParamsEquals(x, publicParams)), 'Duplicate test case params: ' + JSON.stringify(publicParams) ); seen.push(publicParams); @@ -111,39 +134,35 @@ class TestBuilder { return (this as unknown) as TestBuilderWithParams; } - *iterate(rec: TestSpecRecorder): IterableIterator { + *iterate(): IterableIterator { assert(this.testFn !== undefined, 'internal error'); - for (const params of this.cases || [null]) { - yield new RunCaseSpecific(rec, this.name, params, this.fixture, this.testFn); + for (const params of this.cases || [{}]) { + yield new RunCaseSpecific(this.testPath, params, this.fixture, this.testFn); } } } class RunCaseSpecific implements RunCase { readonly id: TestCaseID; - private readonly params: ParamSpec | null; - private readonly recorder: TestSpecRecorder; + + private readonly params: CaseParams | null; private readonly fixture: FixtureClass; private readonly fn: TestFn; constructor( - recorder: TestSpecRecorder, - test: string, - params: ParamSpec | null, + testPath: string[], + params: CaseParams, fixture: FixtureClass, fn: TestFn ) { - this.id = { test, params: params ? extractPublicParams(params) : null }; + this.id = { test: testPath, params: extractPublicParams(params) }; this.params = params; - this.recorder = recorder; this.fixture = fixture; this.fn = fn; } - async run(debug: boolean): Promise { - const [rec, res] = this.recorder.record(this.id.test, this.id.params); - rec.start(debug); - + async run(rec: TestCaseRecorder): Promise { + rec.start(); try { const inst = new this.fixture(rec, this.params || {}); @@ -162,13 +181,6 @@ class RunCaseSpecific implements RunCase { // or unexpected validation/OOM error from the GPUDevice. rec.threw(ex); } - rec.finish(); - return res; - } - - injectResult(result: LiveTestCaseResult): void { - const [, res] = this.recorder.record(this.id.test, this.id.params); - Object.assign(res, result); } } diff --git a/src/common/framework/test_suite_listing.ts b/src/common/framework/test_suite_listing.ts new file mode 100644 index 000000000000..492637d981e4 --- /dev/null +++ b/src/common/framework/test_suite_listing.ts @@ -0,0 +1,16 @@ +// A listing of all specs within a single suite. This is the (awaited) type of +// `groups` in '{cts,unittests}/listing.ts' and `listing` in the auto-generated +// 'out/{cts,unittests}/listing.js' files (see tools/gen_listings). +export type TestSuiteListing = Iterable; + +export type TestSuiteListingEntry = TestSuiteListingEntrySpec | TestSuiteListingEntryReadme; + +interface TestSuiteListingEntrySpec { + readonly file: string[]; + readonly description: string; +} + +interface TestSuiteListingEntryReadme { + readonly file: string[]; + readonly readme: string; +} diff --git a/src/common/framework/tree.ts b/src/common/framework/tree.ts index 514fc505a555..a7da3094c845 100644 --- a/src/common/framework/tree.ts +++ b/src/common/framework/tree.ts @@ -1,82 +1,362 @@ -import { Logger } from './logger.js'; -import { stringifyPublicParams } from './params_utils.js'; -import { TestFilterResult } from './test_filter/test_filter_result.js'; -import { RunCase } from './test_group.js'; +import { TestFileLoader } from './file_loader.js'; +import { TestCaseRecorder } from './logging/test_case_recorder.js'; +import { CaseParamsRW } from './params_utils.js'; +import { compareQueries, Ordering } from './query/compare.js'; +import { + TestQuery, + TestQueryMultiCase, + TestQuerySingleCase, + TestQueryMultiFile, + TestQueryMultiTest, +} from './query/query.js'; +import { stringifySingleParam } from './query/stringify_params.js'; +import { RunCase, RunFn } from './test_group.js'; +import { assert } from './util/util.js'; -export interface FilterResultTreeNode { +// `loadTreeForQuery()` loads a TestTree for a given queryToLoad. +// The resulting tree is a linked-list all the way from `suite:*` to queryToLoad, +// and under queryToLoad is a tree containing every case matched by queryToLoad. +// +// `subqueriesToExpand` influences the `collapsible` flag on nodes in the resulting tree. +// A node is considered "collapsible" if none of the subqueriesToExpand is a StrictSubset +// of that node. +// +// In WebKit/Blink-style web_tests, an expectation file marks individual cts.html "variants" as +// "Failure", "Crash", etc. +// By passing in the list of expectations as the subqueriesToExpand, we can programmatically +// subdivide the cts.html "variants" list to be able to implement arbitrarily-fine suppressions +// (instead of having to suppress entire test files, which would lose a lot of coverage). +// +// `iterateCollapsedQueries()` produces the list of queries for the variants list. +// +// Though somewhat complicated, this system has important benefits: +// - Avoids having to suppress entire test files, which would cause large test coverage loss. +// - Minimizes the number of page loads needed for fine-grained suppressions. +// (In the naive case, we could do one page load per test case - but the test suite would +// take impossibly long to run.) +// - Enables developers to put any number of tests in one file as appropriate, without worrying +// about expectation granularity. + +export interface TestSubtree { + readonly query: T; + readonly children: Map; + readonly collapsible: boolean; description?: string; - runCase?: RunCase; - children?: Map; } -// e.g. iteratePath('a/b/c/d', ':') yields ['a/', 'a/b/', 'a/b/c/', 'a/b/c/d:'] -function* iteratePath(path: string, terminator: string): IterableIterator { - const parts = path.split('/'); - if (parts.length > 1) { - let partial = parts[0] + '/'; - yield partial; - for (let i = 1; i < parts.length - 1; ++i) { - partial += parts[i] + '/'; - yield partial; +export interface TestTreeLeaf { + readonly query: TestQuerySingleCase; + readonly run: RunFn; +} + +export type TestTreeNode = TestSubtree | TestTreeLeaf; + +export class TestTree { + readonly root: TestSubtree; + + constructor(root: TestSubtree) { + this.root = root; + } + + iterateCollapsedQueries(): IterableIterator { + return TestTree.iterateSubtreeCollapsedQueries(this.root); + } + + iterateLeaves(): IterableIterator { + return TestTree.iterateSubtreeLeaves(this.root); + } + + toString(): string { + return TestTree.subtreeToString('(root)', this.root, ''); + } + + static *iterateSubtreeCollapsedQueries(subtree: TestSubtree): IterableIterator { + for (const [, child] of subtree.children) { + if ('children' in child && !child.collapsible) { + yield* TestTree.iterateSubtreeCollapsedQueries(child); + } else { + yield child.query; + } } - // Path ends in '/' (so is a README). - if (parts[parts.length - 1] === '') { - return; + } + + static *iterateSubtreeLeaves(subtree: TestSubtree): IterableIterator { + for (const [, child] of subtree.children) { + if ('children' in child) { + yield* TestTree.iterateSubtreeLeaves(child); + } else { + yield child; + } } } - yield path + terminator; -} -export function treeFromFilterResults( - log: Logger, - listing: IterableIterator -): FilterResultTreeNode { - function getOrInsert(n: FilterResultTreeNode, k: string): FilterResultTreeNode { - const children = n.children!; - if (children.has(k)) { - return children.get(k)!; + static subtreeToString(name: string, tree: TestTreeNode, indent: string): string { + const collapsible = 'run' in tree ? '>' : tree.collapsible ? '+' : '-'; + let s = + indent + + `${collapsible} ${JSON.stringify(name)} => ` + + `${tree.query} ${JSON.stringify(tree.query)}`; + if ('children' in tree) { + if (tree.description !== undefined) { + s += indent + `\n | ${JSON.stringify(tree.description)}`; + } + + for (const [name, child] of tree.children) { + s += '\n' + TestTree.subtreeToString(name, child, indent + ' '); + } } - const v = { children: new Map() }; - children.set(k, v); - return v; + return s; } +} - const tree = { children: new Map() }; - for (const f of listing) { - const files = getOrInsert(tree, f.id.suite + ':'); - if (f.id.path === '') { - // This is a suite README. - files.description = f.spec.description.trim(); +// TODO: Consider having subqueriesToExpand actually impact the depth-order of params in the tree. +export async function loadTreeForQuery( + loader: TestFileLoader, + queryToLoad: TestQuery, + subqueriesToExpand: TestQuery[] +): Promise { + const suite = queryToLoad.suite; + const specs = await loader.listing(suite); + + const subqueriesToExpandEntries = Array.from(subqueriesToExpand.entries()); + const seenSubqueriesToExpand: boolean[] = new Array(subqueriesToExpand.length); + seenSubqueriesToExpand.fill(false); + + const isCollapsible = (subquery: TestQuery) => + subqueriesToExpandEntries.every(([i, toExpand]) => { + const ordering = compareQueries(toExpand, subquery); + + // If toExpand == subquery, no expansion is needed (but it's still "seen"). + if (ordering === Ordering.Equal) seenSubqueriesToExpand[i] = true; + return ordering !== Ordering.StrictSubset; + }); + + // L0 = suite-level, e.g. suite:* + // L1 = file-level, e.g. suite:a,b:* + // L2 = test-level, e.g. suite:a,b:c,d:* + // L3 = case-level, e.g. suite:a,b:c,d: + let foundCase = false; + // L0 is suite:* + const subtreeL0 = makeTreeForSuite(suite); + isCollapsible(subtreeL0.query); // mark seenSubqueriesToExpand + for (const entry of specs) { + if (entry.file.length === 0 && 'readme' in entry) { + // Suite-level readme. + assert(subtreeL0.description === undefined); + subtreeL0.description = entry.readme.trim(); continue; } - let tests = files; - for (const path of iteratePath(f.id.path, ':')) { - tests = getOrInsert(tests, f.id.suite + ':' + path); - } - if (f.spec.description) { - // This is a directory README or spec file. - tests.description = f.spec.description.trim(); + { + const queryL1 = new TestQueryMultiFile(suite, entry.file); + const orderingL1 = compareQueries(queryL1, queryToLoad); + if (orderingL1 === Ordering.Unordered) { + // File path is not matched by this query. + continue; + } } - if (!('g' in f.spec)) { - // This is a directory README. + if ('readme' in entry) { + // Entry is a README that is an ancestor or descendant of the query. + // (It's included for display in the standalone runner.) + + // readmeSubtree is suite:a,b,* + // (This is always going to dedup with a file path, if there are any test spec files under + // the directory that has the README). + const readmeSubtree: TestSubtree = addSubtreeForDirPath( + subtreeL0, + entry.file + ); + assert(readmeSubtree.description === undefined); + readmeSubtree.description = entry.readme.trim(); continue; } + // Entry is a spec file. - const [tRec] = log.record(f.id); - const fId = f.id.suite + ':' + f.id.path; - for (const t of f.spec.g.iterate(tRec)) { - let cases = tests; - for (const path of iteratePath(t.id.test, '~')) { - cases = getOrInsert(cases, fId + ':' + path); + const spec = await loader.importSpecFile(queryToLoad.suite, entry.file); + const description = spec.description.trim(); + // subtreeL1 is suite:a,b:* + const subtreeL1: TestSubtree = addSubtreeForFilePath( + subtreeL0, + entry.file, + description, + isCollapsible + ); + + // TODO: If tree generation gets too slow, avoid actually iterating the cases in a file + // if there's no need to (based on the subqueriesToExpand). + for (const t of spec.g.iterate()) { + { + const queryL3 = new TestQuerySingleCase(suite, entry.file, t.id.test, t.id.params); + const orderingL3 = compareQueries(queryL3, queryToLoad); + if (orderingL3 === Ordering.Unordered || orderingL3 === Ordering.StrictSuperset) { + // Case is not matched by this query. + continue; + } } - const p = stringifyPublicParams(t.id.params); - cases.children!.set(fId + ':' + t.id.test + '=' + p, { - runCase: t, - }); + // subtreeL2 is suite:a,b:c,d:* + const subtreeL2: TestSubtree = addSubtreeForTestPath( + subtreeL1, + t.id.test, + isCollapsible + ); + + // Leaf for case is suite:a,b:c,d:x=1;y=2 + addLeafForCase(subtreeL2, t, isCollapsible); + + foundCase = true; } } + const tree = new TestTree(subtreeL0); + + for (const [i, sq] of subqueriesToExpandEntries) { + const seen = seenSubqueriesToExpand[i]; + assert( + seen, + `subqueriesToExpand entry did not match anything \ +(can happen due to overlap with another subquery): ${sq.toString()}` + ); + } + assert(foundCase, 'Query does not match any cases'); + + // TODO: Contains lots of single-child subtrees. Consider cleaning those up (as postprocess?). + return tree; +} + +function makeTreeForSuite(suite: string): TestSubtree { + return { + query: new TestQueryMultiFile(suite, []), + children: new Map(), + collapsible: false, + }; +} + +function addSubtreeForDirPath( + tree: TestSubtree, + file: string[] +): TestSubtree { + const subqueryFile: string[] = []; + // To start, tree is suite:* + // This loop goes from that -> suite:a,* -> suite:a,b,* + for (const part of file) { + subqueryFile.push(part); + tree = getOrInsertSubtree(part, tree, () => { + const query = new TestQueryMultiFile(tree.query.suite, subqueryFile); + return { query, collapsible: false }; + }); + } return tree; } + +function addSubtreeForFilePath( + tree: TestSubtree, + file: string[], + description: string, + checkCollapsible: (sq: TestQuery) => boolean +): TestSubtree { + // To start, tree is suite:* + // This goes from that -> suite:a,* -> suite:a,b,* + tree = addSubtreeForDirPath(tree, file); + // This goes from that -> suite:a,b:* + const subtree = getOrInsertSubtree('', tree, () => { + const query = new TestQueryMultiTest(tree.query.suite, tree.query.filePathParts, []); + return { query, description, collapsible: checkCollapsible(query) }; + }); + return subtree; +} + +function addSubtreeForTestPath( + tree: TestSubtree, + test: readonly string[], + isCollapsible: (sq: TestQuery) => boolean +): TestSubtree { + const subqueryTest: string[] = []; + // To start, tree is suite:a,b:* + // This loop goes from that -> suite:a,b:c,* -> suite:a,b:c,d,* + for (const part of test) { + subqueryTest.push(part); + tree = getOrInsertSubtree(part, tree, () => { + const query = new TestQueryMultiTest( + tree.query.suite, + tree.query.filePathParts, + subqueryTest + ); + return { query, collapsible: isCollapsible(query) }; + }); + } + // This goes from that -> suite:a,b:c,d:* + return getOrInsertSubtree('', tree, () => { + const query = new TestQueryMultiCase( + tree.query.suite, + tree.query.filePathParts, + subqueryTest, + {} + ); + return { query, collapsible: isCollapsible(query) }; + }); +} + +function addLeafForCase( + tree: TestSubtree, + t: RunCase, + checkCollapsible: (sq: TestQuery) => boolean +): void { + const query = tree.query; + let name: string = ''; + const subqueryParams: CaseParamsRW = {}; + + // To start, tree is suite:a,b:c,d:* + // This loop goes from that -> suite:a,b:c,d:x=1;* -> suite:a,b:c,d:x=1;y=2;* + for (const [k, v] of Object.entries(t.id.params)) { + name = stringifySingleParam(k, v); + subqueryParams[k] = v; + + tree = getOrInsertSubtree(name, tree, () => { + const subquery = new TestQueryMultiCase( + query.suite, + query.filePathParts, + query.testPathParts, + subqueryParams + ); + return { query: subquery, collapsible: checkCollapsible(subquery) }; + }); + } + + // This goes from that -> suite:a,b:c,d:x=1;y=2 + const subquery = new TestQuerySingleCase( + query.suite, + query.filePathParts, + query.testPathParts, + subqueryParams + ); + checkCollapsible(subquery); // mark seenSubqueriesToExpand + insertLeaf(tree, subquery, t); +} + +function getOrInsertSubtree( + key: string, + parent: TestSubtree, + createSubtree: () => Omit, 'children'> +): TestSubtree { + let v: TestSubtree; + const child = parent.children.get(key); + if (child !== undefined) { + assert('children' in child); // Make sure cached subtree is not actually a leaf + v = child as TestSubtree; + } else { + v = { ...createSubtree(), children: new Map() }; + parent.children.set(key, v); + } + return v; +} + +function insertLeaf(parent: TestSubtree, query: TestQuerySingleCase, t: RunCase) { + const key = ''; + const leaf: TestTreeLeaf = { + query, + run: (rec: TestCaseRecorder) => t.run(rec), + }; + assert(!parent.children.has(key)); + parent.children.set(key, leaf); +} diff --git a/src/common/framework/url_query.ts b/src/common/framework/url_query.ts deleted file mode 100644 index b963ce3277c7..000000000000 --- a/src/common/framework/url_query.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { TestCaseID, TestSpecID } from './id.js'; -import { ParamArgument, stringifyPublicParams } from './params_utils.js'; -import { unreachable } from './util/util.js'; - -export function encodeSelectively(s: string): string { - let ret = encodeURIComponent(s); - ret = ret.replace(/%22/g, '"'); - ret = ret.replace(/%2C/g, ','); - ret = ret.replace(/%2F/g, '/'); - ret = ret.replace(/%3A/g, ':'); - ret = ret.replace(/%3D/g, '='); - ret = ret.replace(/%5B/g, '['); - ret = ret.replace(/%5D/g, ']'); - ret = ret.replace(/%7B/g, '{'); - ret = ret.replace(/%7D/g, '}'); - return ret; -} - -export function checkPublicParamType(v: ParamArgument): void { - if (typeof v === 'number' || typeof v === 'string' || typeof v === 'boolean' || v === undefined) { - return; - } - if (v instanceof Array) { - for (const x of v) { - if (typeof x !== 'number') { - break; - } - } - return; - } - unreachable('Invalid type for test case params ' + v); -} - -export function makeQueryString(spec: TestSpecID, testcase?: TestCaseID): string { - let s = spec.suite + ':'; - s += spec.path + ':'; - if (testcase !== undefined) { - s += testcase.test + '='; - s += stringifyPublicParams(testcase.params); - } - return encodeSelectively(s); -} diff --git a/src/common/framework/collect_garbage.ts b/src/common/framework/util/collect_garbage.ts similarity index 95% rename from src/common/framework/collect_garbage.ts rename to src/common/framework/util/collect_garbage.ts index 11bc6e7dd7e8..d18a24095003 100644 --- a/src/common/framework/collect_garbage.ts +++ b/src/common/framework/util/collect_garbage.ts @@ -1,4 +1,4 @@ -import { resolveOnTimeout } from './util/util.js'; +import { resolveOnTimeout } from './util.js'; /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ declare const Components: any; diff --git a/src/common/framework/util/stack.ts b/src/common/framework/util/stack.ts index 573036bb3832..583ea4b811cd 100644 --- a/src/common/framework/util/stack.ts +++ b/src/common/framework/util/stack.ts @@ -1,33 +1,17 @@ -// Takes a stack trace, and extracts only the first continuous range of lines -// containing '/(webgpu|unittests)/', which should provide only the useful part -// of the stack to the caller (for logging). -export function getStackTrace(e: Error): string { +// Returns the stack trace of an Error, but without the extra boilerplate at the bottom +// (e.g. RunCaseSpecific, processTicksAndRejections, etc.), for logging. +export function extractImportantStackTrace(e: Error): string { if (!e.stack) { return ''; } - - const parts = e.stack.split('\n'); - - const stack = []; - const moreStack = []; - let found = false; - const commonRegex = /[/\\](webgpu|unittests)[/\\]/; - for (let i = 0; i < parts.length; ++i) { - const part = parts[i].trim(); - const isSuites = commonRegex.test(part); // approximate - if (found && !isSuites) { - moreStack.push(part); - } - if (isSuites) { - if (moreStack.length) { - stack.push(...moreStack); - moreStack.length = 0; - } - stack.push(part); - found = true; + const lines = e.stack.split('\n'); + for (let i = lines.length - 1; i >= 0; --i) { + const line = lines[i]; + if (line.indexOf('.spec.') !== -1) { + return lines.slice(0, i + 1).join('\n'); } } - return stack.join('\n'); + return e.stack; } // *** Examples *** diff --git a/src/common/runtime/cmdline.ts b/src/common/runtime/cmdline.ts index d9d2db356abf..f6b5965ef6ac 100644 --- a/src/common/runtime/cmdline.ts +++ b/src/common/runtime/cmdline.ts @@ -4,10 +4,9 @@ import * as fs from 'fs'; import * as process from 'process'; -import { TestSpecID } from '../framework/id.js'; -import { TestLoader } from '../framework/loader.js'; -import { LiveTestCaseResult, Logger } from '../framework/logger.js'; -import { makeQueryString } from '../framework/url_query.js'; +import { DefaultTestFileLoader } from '../framework/file_loader.js'; +import { Logger } from '../framework/logging/logger.js'; +import { LiveTestCaseResult } from '../framework/logging/result.js'; import { assert, unreachable } from '../framework/util/util.js'; function usage(rc: number): never { @@ -29,7 +28,7 @@ if (!fs.existsSync('src/common/runtime/cmdline.ts')) { let verbose = false; let debug = false; let printJSON = false; -const filterArgs = []; +const filterArgs: string[] = []; for (const a of process.argv.slice(2)) { if (a.startsWith('-')) { if (a === '--verbose') { @@ -52,44 +51,42 @@ if (filterArgs.length === 0) { (async () => { try { - const loader = new TestLoader(); - const files = await loader.loadTestsFromCmdLine(filterArgs); + const loader = new DefaultTestFileLoader(); + assert(filterArgs.length === 1, 'currently, there must be exactly one query on the cmd line'); + const testcases = await loader.loadTests(filterArgs[0]); - const log = new Logger(); + const log = new Logger(debug); - const failed: Array<[TestSpecID, LiveTestCaseResult]> = []; - const warned: Array<[TestSpecID, LiveTestCaseResult]> = []; - const skipped: Array<[TestSpecID, LiveTestCaseResult]> = []; + const failed: Array<[string, LiveTestCaseResult]> = []; + const warned: Array<[string, LiveTestCaseResult]> = []; + const skipped: Array<[string, LiveTestCaseResult]> = []; let total = 0; - for (const f of files) { - if (!('g' in f.spec)) { - continue; + + for (const testcase of testcases) { + const name = testcase.query.toString(); + const [rec, res] = log.record(name); + await testcase.run(rec); + + if (verbose) { + printResults([[name, res]]); } - const [rec] = log.record(f.id); - for (const t of f.spec.g.iterate(rec)) { - const res = await t.run(debug); - if (verbose) { - printResults([[f.id, res]]); - } - - total++; - switch (res.status) { - case 'pass': - break; - case 'fail': - failed.push([f.id, res]); - break; - case 'warn': - warned.push([f.id, res]); - break; - case 'skip': - skipped.push([f.id, res]); - break; - default: - unreachable('unrecognized status'); - } + total++; + switch (res.status) { + case 'pass': + break; + case 'fail': + failed.push([name, res]); + break; + case 'warn': + warned.push([name, res]); + break; + case 'skip': + skipped.push([name, res]); + break; + default: + unreachable('unrecognized status'); } } @@ -138,9 +135,9 @@ Failed = ${rpt(failed.length)}`); } })(); -function printResults(results: Array<[TestSpecID, LiveTestCaseResult]>): void { - for (const [id, r] of results) { - console.log(`[${r.status}] ${makeQueryString(id, r)} (${r.timems}ms). Log:`); +function printResults(results: Array<[string, LiveTestCaseResult]>): void { + for (const [name, r] of results) { + console.log(`[${r.status}] ${name} (${r.timems}ms). Log:`); if (r.logs) { for (const l of r.logs) { console.log(' - ' + l.toJSON().replace(/\n/g, '\n ')); diff --git a/src/common/runtime/helper/test_worker-worker.ts b/src/common/runtime/helper/test_worker-worker.ts index 395493472508..5bb742cd678e 100644 --- a/src/common/runtime/helper/test_worker-worker.ts +++ b/src/common/runtime/helper/test_worker-worker.ts @@ -1,28 +1,24 @@ -import { TestLoader } from '../../framework/loader.js'; -import { Logger } from '../../framework/logger.js'; +import { DefaultTestFileLoader } from '../../framework/file_loader.js'; +import { Logger } from '../../framework/logging/logger.js'; import { assert } from '../../framework/util/util.js'; /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ declare const self: any; // should be DedicatedWorkerGlobalScope -const log = new Logger(); -const loader = new TestLoader(); +const loader = new DefaultTestFileLoader(); self.onmessage = async (ev: MessageEvent) => { - const { query, debug } = ev.data; + const query: string = ev.data.query; + const debug: boolean = ev.data.debug; - const files = Array.from(await loader.loadTests([query])); - assert(files.length === 1, 'worker query resulted in != 1 files'); + const log = new Logger(debug); - const f = files[0]; - const [rec] = log.record(f.id); - assert('g' in f.spec, 'worker query resulted in README'); + const testcases = Array.from(await loader.loadTests(query)); + assert(testcases.length === 1, 'worker query resulted in != 1 cases'); - const cases = Array.from(f.spec.g.iterate(rec)); - assert(cases.length === 1, 'worker query resulted in != 1 cases'); - const c = cases[0]; - - const result = await c.run(debug); + const testcase = testcases[0]; + const [rec, result] = log.record(testcase.query.toString()); + await testcase.run(rec); self.postMessage({ query, result }); }; diff --git a/src/common/runtime/helper/test_worker.ts b/src/common/runtime/helper/test_worker.ts index 251a405ed732..cb526a0b43da 100644 --- a/src/common/runtime/helper/test_worker.ts +++ b/src/common/runtime/helper/test_worker.ts @@ -1,14 +1,15 @@ -import { - LiveTestCaseResult, - LogMessageWithStack, - TransferredTestCaseResult, -} from '../../framework/logger.js'; +import { LogMessageWithStack } from '../../framework/logging/log_message.js'; +import { TransferredTestCaseResult, LiveTestCaseResult } from '../../framework/logging/result.js'; +import { TestCaseRecorder } from '../../framework/logging/test_case_recorder.js'; export class TestWorker { - private worker: Worker; - private resolvers = new Map void>(); + private readonly debug: boolean; + private readonly worker: Worker; + private readonly resolvers = new Map void>(); + + constructor(debug: boolean) { + this.debug = debug; - constructor() { const selfPath = import.meta.url; const selfPathDir = selfPath.substring(0, selfPath.lastIndexOf('/')); const workerPath = selfPathDir + '/test_worker-worker.js'; @@ -28,10 +29,11 @@ export class TestWorker { }; } - run(query: string, debug: boolean = false): Promise { - this.worker.postMessage({ query, debug }); - return new Promise(resolve => { + async run(rec: TestCaseRecorder, query: string): Promise { + this.worker.postMessage({ query, debug: this.debug }); + const workerResult = await new Promise(resolve => { this.resolvers.set(query, resolve); }); + rec.injectResult(workerResult); } } diff --git a/src/common/runtime/standalone.ts b/src/common/runtime/standalone.ts index 2fb963b2a161..a21607157aa6 100644 --- a/src/common/runtime/standalone.ts +++ b/src/common/runtime/standalone.ts @@ -1,10 +1,10 @@ // Implements the standalone test runner (see also: /standalone/index.html). -import { TestLoader } from '../framework/loader.js'; -import { LiveTestCaseResult, Logger } from '../framework/logger.js'; -import { RunCase } from '../framework/test_group.js'; -import { FilterResultTreeNode, treeFromFilterResults } from '../framework/tree.js'; -import { encodeSelectively } from '../framework/url_query.js'; +import { DefaultTestFileLoader } from '../framework/file_loader.js'; +import { Logger } from '../framework/logging/logger.js'; +import { TestQuery } from '../framework/query/query.js'; +import { TestTreeNode, TestSubtree, TestTreeLeaf } from '../framework/tree.js'; +import { assert } from '../framework/util/util.js'; import { optionEnabled } from './helper/options.js'; import { TestWorker } from './helper/test_worker.js'; @@ -15,12 +15,13 @@ window.onbeforeunload = () => { }; let haveSomeResults = false; -const log = new Logger(); const runnow = optionEnabled('runnow'); const debug = optionEnabled('debug'); -const worker = optionEnabled('worker') ? new TestWorker() : undefined; +const logger = new Logger(debug); + +const worker = optionEnabled('worker') ? new TestWorker(debug) : undefined; const resultsVis = document.getElementById('resultsVis')!; const resultsJSON = document.getElementById('resultsJSON')!; @@ -29,25 +30,25 @@ type RunSubtree = () => Promise; // DOM generation -function makeTreeNodeHTML(name: string, tree: FilterResultTreeNode): [HTMLElement, RunSubtree] { - if (tree.children) { - return makeSubtreeHTML(name, tree); +function makeTreeNodeHTML(tree: TestTreeNode): [HTMLElement, RunSubtree] { + if ('children' in tree) { + return makeSubtreeHTML(tree); } else { - return makeCaseHTML(name, tree.runCase!); + return makeCaseHTML(tree); } } -function makeCaseHTML(name: string, t: RunCase): [HTMLElement, RunSubtree] { +function makeCaseHTML(t: TestTreeLeaf): [HTMLElement, RunSubtree] { const div = $('
').addClass('testcase'); + const name = t.query.toString(); const runSubtree = async () => { haveSomeResults = true; - let res: LiveTestCaseResult; + const [rec, res] = logger.record(name); if (worker) { - res = await worker.run(name, debug); - t.injectResult(res); + await worker.run(rec, name); } else { - res = await t.run(debug); + await t.run(rec); } casetime.text(res.timems.toFixed(4) + ' ms'); @@ -72,7 +73,7 @@ function makeCaseHTML(name: string, t: RunCase): [HTMLElement, RunSubtree] { } }; - const casehead = makeTreeNodeHeaderHTML(name, undefined, runSubtree); + const casehead = makeTreeNodeHeaderHTML(t.query, undefined, runSubtree, true); div.append(casehead); const casetime = $('
').addClass('testcasetime').html('ms').appendTo(casehead); const caselogs = $('
').addClass('testcaselogs').appendTo(div); @@ -80,27 +81,22 @@ function makeCaseHTML(name: string, t: RunCase): [HTMLElement, RunSubtree] { return [div[0], runSubtree]; } -function makeSubtreeHTML(name: string, subtree: FilterResultTreeNode): [HTMLElement, RunSubtree] { +function makeSubtreeHTML(t: TestSubtree): [HTMLElement, RunSubtree] { const div = $('
').addClass('subtree'); - const header = makeTreeNodeHeaderHTML(name, subtree.description, () => { - return runSubtree(); - }); + const header = makeTreeNodeHeaderHTML(t.query, t.description, () => runSubtree(), false); div.append(header); const subtreeHTML = $('
').addClass('subtreechildren').appendTo(div); - const runSubtree = makeSubtreeChildrenHTML(subtreeHTML[0], subtree.children!); + const runSubtree = makeSubtreeChildrenHTML(subtreeHTML[0], t.children.values()); return [div[0], runSubtree]; } -function makeSubtreeChildrenHTML( - div: HTMLElement, - children: Map -): RunSubtree { +function makeSubtreeChildrenHTML(div: HTMLElement, children: Iterable): RunSubtree { const runSubtreeFns: RunSubtree[] = []; - for (const [name, subtree] of children) { - const [subtreeHTML, runSubtree] = makeTreeNodeHTML(name, subtree); + for (const subtree of children) { + const [subtreeHTML, runSubtree] = makeTreeNodeHTML(subtree); div.append(subtreeHTML); runSubtreeFns.push(runSubtree); } @@ -113,24 +109,16 @@ function makeSubtreeChildrenHTML( } function makeTreeNodeHeaderHTML( - name: string, + query: TestQuery, description: string | undefined, - runSubtree: RunSubtree + runSubtree: RunSubtree, + isLeaf: boolean ): HTMLElement { const div = $('
').addClass('nodeheader'); - const nameEncoded = encodeSelectively(name); - let nameHTML; - { - const i = nameEncoded.indexOf('{'); - const n1 = i === -1 ? nameEncoded : nameEncoded.slice(0, i + 1); - const n2 = i === -1 ? '' : nameEncoded.slice(i + 1); - nameHTML = n1.replace(/:/g, ':') + '' + n2.replace(/,/g, ','); - } - - const href = `?${worker ? 'worker&' : ''}${debug ? 'debug&' : ''}q=${nameEncoded}`; + const href = `?${worker ? 'worker&' : ''}${debug ? 'debug&' : ''}q=${query.toString()}`; $('