diff --git a/package.json b/package.json index a21aaa5..121da61 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "yalc": "~1.0.0-pre.21" }, "dependencies": { - "@tensorflow/tfjs-core": "1.2.0" + "@tensorflow/tfjs-core": "1.2.0", + "@tensorflow/tfjs-layers": "^1.2.1" } } diff --git a/src/benchmark_ops_test.ts b/src/benchmark_ops_test.ts index 7b7e29c..eeefc6f 100644 --- a/src/benchmark_ops_test.ts +++ b/src/benchmark_ops_test.ts @@ -1,69 +1,86 @@ -/** - * @license - * Copyright 2019 Google LLC. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================================= - */ - -import * as tf from '@tensorflow/tfjs-core'; -import * as tfwebgl2compute from './index'; - -describe('Ops benchmarks', () => { - beforeAll(async () => await tfwebgl2compute.ready); - - it('matMul', async () => { - const times = []; - - const a = tf.randomNormal([500, 500]); - const b = tf.randomNormal([500, 500]); - - let c = tf.matMul(a, b); - await c.data(); - - for (let i = 0; i < 100; i++) { - const start = performance.now(); - c = tf.matMul(a, b); - await c.data(); - times.push(performance.now() - start); - } - - a.dispose(); - b.dispose(); - console.log(`MatMul: Average time ms: ${ - times.reduce((a, b) => a + b, 0) / times.length}`); - console.log(`Min time ms: ${Math.min(...times)}`); - }); - - it('conv2d', async () => { - const times = []; - - const a = tf.randomNormal([1, 128, 128, 4]); - const b = tf.randomNormal([25, 25, 4, 4]); - - let c = tf.conv2d(a, b, 1, 'same'); - await c.data(); - - for (let i = 0; i < 100; i++) { - const start = performance.now(); - c = tf.conv2d(a, b, 1, 'same'); - await c.data(); - times.push(performance.now() - start); - } - - a.dispose(); - b.dispose(); - console.log(`Conv2d: Average time ms: ${ - times.reduce((a, b) => a + b, 0) / times.length}`); - console.log(`Min time ms: ${Math.min(...times)}`); - }); +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +import * as tf from '@tensorflow/tfjs-core'; +import * as tfwebgl2compute from './index'; +import {MobileNetV1GPUBenchmark} from './mobilenet_benchmarks'; +import * as test_util from './test_util'; + +describe('Ops benchmarks', () => { + beforeAll(async () => { + await tfwebgl2compute.ready; + jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000; + }); + + it('mobilenet_v1', async () => { + const sizes = [1]; // MobileNet version + const runs = 20; + + const benchmark = new MobileNetV1GPUBenchmark(); + await benchmark.loadModel(); + + await test_util.benchmarkAndLog( + 'mobilenet_v1', size => benchmark.run(size), sizes, + size => `N=${size}_0_224`, runs); + }); + + it('matMul', async () => { + const times = []; + + const a = tf.randomNormal([500, 500]); + const b = tf.randomNormal([500, 500]); + + let c = tf.matMul(a, b); + await c.data(); + + for (let i = 0; i < 100; i++) { + const start = performance.now(); + c = tf.matMul(a, b); + await c.data(); + times.push(performance.now() - start); + } + + a.dispose(); + b.dispose(); + console.log(`MatMul: Average time ms: ${ + times.reduce((a, b) => a + b, 0) / times.length}`); + console.log(`Min time ms: ${Math.min(...times)}`); + }); + + it('conv2d', async () => { + const times = []; + + const a = tf.randomNormal([1, 128, 128, 4]); + const b = tf.randomNormal([25, 25, 4, 4]); + + let c = tf.conv2d(a, b, 1, 'same'); + await c.data(); + + for (let i = 0; i < 100; i++) { + const start = performance.now(); + c = tf.conv2d(a, b, 1, 'same'); + await c.data(); + times.push(performance.now() - start); + } + + a.dispose(); + b.dispose(); + console.log(`Conv2d: Average time ms: ${ + times.reduce((a, b) => a + b, 0) / times.length}`); + console.log(`Min time ms: ${Math.min(...times)}`); + }); }); \ No newline at end of file diff --git a/src/mobilenet_benchmarks.ts b/src/mobilenet_benchmarks.ts new file mode 100644 index 0000000..140a473 --- /dev/null +++ b/src/mobilenet_benchmarks.ts @@ -0,0 +1,47 @@ +/** + * @license + * Copyright 2018 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import * as tfc from '@tensorflow/tfjs-core'; +import * as tfl from '@tensorflow/tfjs-layers'; + +import {BenchmarkModelTest} from './types'; +import * as util from './util'; + +const MOBILENET_MODEL_PATH = + // tslint:disable-next-line:max-line-length + 'https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_1.0_224/model.json'; + +export class MobileNetV1GPUBenchmark implements BenchmarkModelTest { + private model: tfl.LayersModel; + + async loadModel() { + this.model = await tfl.loadLayersModel(MOBILENET_MODEL_PATH); + } + + async run(size: number): Promise { + tfc.setBackend('webgl'); + + const zeros = tfc.zeros([1, 224, 224, 3]); + + const benchmark = () => this.model.predict(zeros); + + const time = await util.benchmark(benchmark); + + zeros.dispose(); + + return time; + } +} diff --git a/src/setup_test.ts b/src/setup_test.ts index 344d4e5..5fc1c6f 100644 --- a/src/setup_test.ts +++ b/src/setup_test.ts @@ -126,4 +126,4 @@ env.specFilter = spec => { }; // Import and run all the tests from core. -import '@tensorflow/tfjs-core/dist/tests'; +//import '@tensorflow/tfjs-core/dist/tests'; diff --git a/src/test_util.ts b/src/test_util.ts new file mode 100644 index 0000000..e671a06 --- /dev/null +++ b/src/test_util.ts @@ -0,0 +1,51 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +//import * as firebase from './firebase'; +import {BenchmarkLog} from './types'; + +function nextTick(): Promise { + return new Promise(resolve => setTimeout(resolve)); +} + +// tslint:disable-next-line:no-any +export async function benchmarkAndLog( + name: string, benchmark: (size: T) => Promise, sizes: T[], + sizeToParams: (size: T) => string, runCount = 100, + warmupRunCount = 1): Promise { + const logs: BenchmarkLog[] = []; + + for (let i = 0; i < sizes.length; i++) { + const size = sizes[i]; + let averageTimeMs = 0; + let result; + + for (let j = 0; j < warmupRunCount; j++) { + result = await benchmark(size); + } + + for (let j = 0; j < runCount; j++) { + result = await benchmark(size); + averageTimeMs += result / runCount; + await nextTick(); + } + const benchmarkLog: + BenchmarkLog = {params: sizeToParams(size), averageTimeMs}; + logs.push(benchmarkLog); + } + //await firebase.logBenchmarkRun(name, logs); +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..a5aa7f0 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,332 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * ======================================================= + * Old types used by the tjfs-core benchmarks. + * ======================================================= + */ + +export interface BenchmarkTest { + run(size: number, opType?: string, params?: {}): Promise; +} + +export interface BenchmarkModelTest extends BenchmarkTest { + loadModel(): void; +} + +export interface BenchmarkLog { + averageTimeMs: number; + params: string; +} + +/** + * ======================================================= + * New types to be used by tfjs-layers and tfjs-node + * benchmarks. + * + * We plan to migrate the tfjs-core benchmarks to the new + * type system as well. + * ======================================================= + */ + +/** + * Interfaces that correspond to collections in Firestore Collections. + */ + +export type VersionSetCollection = { + [versionSetId: string]: VersionSet +}; + +export type EnvironmentCollection = { + [environmentId: string]: EnvironmentInfo +}; + +/** + * Used by the dashboard front end to quickly retrieve a list + * of available benchmark tasks. + * + * New documents (rows) will be added to this collection (table) only when a + * new benchmark task (e.g., a new model, or a new function of an existing + * model) has been added. This is how a client frontend knows what models and + * function (and non-model tasks) are available in the database. + * Therefore, multiple runs of the same task will reuse the same document (row) + * in this collection (table). + * + * Note: this collection has redundant information with `BenchmarkRun` and + * `BenchmarkRunCollection`. It is essentially an index that speeds up queries + * for all available tasks. In principle, this collection can be recreated + * from `BenchmarkRunCollection` if necessary. + */ +export type TaskCollection = { + [taskId: string]: Task +}; + +/** + * A collection containing information about benchmarked models. + * + * Their versions and description. + * + * This applies to only model-based benchmarks and does not apply to + * non-model benchmarks. + */ +export type ModelCollection = { + [modelId: string]: Model +}; + +/** The collection that stores the actual benchmark results data. */ +export type BenchmarkRunCollection = { + [benchmarkRunId: string]: BenchmarkRun +}; + +/** Version sets. */ + +export type CodeRepository = + 'tfjs'|'tfjs-converter'|'tfjs-core'|'tfjs-data'|'tfjs-layers'|'tfjs-node'; + +export interface VersionSet { + commitHashes?: {[repo in CodeRepository]?: string}; + versions?: {[repo in CodeRepository]?: string}; +} + +/** Environments. */ + +export type BrowserEnvironmentType = + 'chrome-linux'|'chrome-mac'|'firefox-linux'|'firefox-mac'|'safari-mac'| + 'chrome-windows'|'firefox-windows'|'chrome-ios-11'|'safari-ios-11'; +export type NodeEnvironmentType = + 'node-libtensorflow-cpu'|'node-libtensorflow-cuda'|'node-gles'; +export type PythonEnvironmentType = + 'python-tensorflow-cpu'|'python-tensorflow-cuda'; +export type ServerSideEnvironmentType = + NodeEnvironmentType|PythonEnvironmentType; +export type BenchmarkEnvironmentType = + BrowserEnvironmentType|ServerSideEnvironmentType; + +export interface EnvironmentInfo { + type: BenchmarkEnvironmentType; + + /** `uname -a` output. */ + systemInfo?: string; + + /** `inxi` output. */ + cpuInfo?: string; + + /** Processed `free` output. */ + memInfo?: string; +} + +export interface BrowserEnvironmentInfo extends EnvironmentInfo { + type: BrowserEnvironmentType; + + /** Browser `navigator.userAgent`. */ + userAgent: string; + + webGLVersion?: string; +} + +export interface ServerSideEnvironmentInfo extends EnvironmentInfo { + type: ServerSideEnvironmentType; + + /** Processed output from `nvidia-smi`. */ + cudaGPUInfo?: string; + + /** Output from `nvcc --version`. */ + cudaVersion?: string; +} + +export interface NodeEnvironmentInfo extends ServerSideEnvironmentInfo { + type: NodeEnvironmentType; + + /** `node --version` output. */ + nodeVersion: string; + tfjsNodeVersion?: string; + tfjsNodeUsesCUDA?: boolean; +} + +export interface PythonEnvironmentInfo extends ServerSideEnvironmentInfo { + type: PythonEnvironmentType; + + /** Output of `python --version`. */ + pythonVersion?: string; + + tensorflowVersion?: string; + kerasVersion?: string; + tensorflowUsesCUDA?: boolean; +} + +/** + * Task types, names and function names under each task. + * + * If a task is performed in multiple environments (e.g., in tfjs-node and + * Python), they should correspond the same `Task` object. + */ +export interface Task { + taskType: TaskType; + + /** + * Name of the task. + * + * For model-based tasks, this is the name of the model. + */ + taskName: string; + + /** + * For model-based tasks, this is the function name, e.g., + * predict(), fit(), fitDataset(). + */ + functionName?: string; +} + +export interface ModelTask extends Task { + /** A reference to Model in ModelCollection. */ + modelId: string; + + functionName: ModelFunctionName; +} + +/** Information about a benchmarked model. */ +export type ModelType = + 'keras-model'|'tensorflow-model'|'tfjs-layers-model'|'tfjs-graph-model'; + +export interface Model { + type: ModelType; + + name: string; + + version: string; + + description: string; +} + +/** Benchmark tasks logs. */ + +// TODO(cais): Add new types in the future, such as low-level tensor +// operations, etc. +export type TaskType = 'model'; + +export type ModelFormat = 'LayersModel'|'GraphModel'; + +export type ModelFunctionName = 'predict'|'fit'|'fitDataset'; + +export type FunctionName = ModelFunctionName; + +/** + * The results and related data from a benchmark run. + * + * A benchmark run is a full set of iterations that assess the performance + * of a specific task (see `Task` interface) in a given environment, + * under a given version / commit set. It includes a number of warm-up + * (a.k.a., burn-in) iterations, followed by a number of benchmarked + * iterations. + * + * See the doc strings below for what an iteration means in the context + * of Model.predict(), fit(), fitDataset() and non-model-based tasks. + */ +export interface BenchmarkRun { + versionSetId: string; + environmentId: string; + taskId: string; + + taskType: TaskType; + + functionName: string; + + /** + * Number of burn-in (i.e., warm-up) iterations that take place prior to the + * benchmarked iterations. + * + * Follows the same convention as `numBenchmarkedIterations` for counting + * iterations. + */ + numWarmUpIterations: number; + + /** + * Number of individual iterations that are timed. + * + * - For predict() and non-model-based tasks, this is the number of function + * calls. + * - For fit() and fitDatset(), this is the number of epochs of a single call. + */ + numBenchmarkedIterations: number; + + /** The ending timestamp of the task, in milliseconds since epoch. */ + endingTimestampMs: number; + + /** + * Raw benchmarked run times in milliseconds. + * + * This field is optional because for certain types of benchmarks (e.g., + * tf.LayersModel.fit() calls), the individual-iteration times are not + * available (e.g., cannot obtain epoch-by-epoch times without using + * a callback; but a callback affects the timing itself.) + * However, in cases where the individual-iteration times are available + * (e.g., tf.LayersModel.predict calls), it should be populated. + * + * - For predict() and non-model-based tasks, each item of the array is + * the wall time of a single predict() call or some other type of function + * call. + * - For fit() and fitDatset(), this field is not populated, due to the + * fact that obtaining per-epoch time requires a callback and that may + * affect the performance of the fit() / fitDataset() call. + */ + timesMs?: number[]; + + /** + * Arithmetic mean of `benchmarkedRunTimesMs`. + * + * - For predict() and non-model-based tasks, this is the arithmetic mean + * of `timeMs`. + * - For a fit() or fitDataset() task, this is the total time of the function + * call divided by the number of training epochs (i.e., + * `numBenchmarkedIterations`). + */ + averageTimeMs: number; +} + +export interface ModelBenchmarkRun extends BenchmarkRun { + taskType: 'model'; + + modelFormat: ModelFormat; + + modelName: string; + + functionName: ModelFunctionName; + + batchSize: number; +} + +export interface ModelTrainingBenchmarkRun extends ModelBenchmarkRun { + loss: string; + optimizer: string; +} + +export interface TensorDef { + value: number[]; + shape: number[]; + dtype: 'string'|'float32'|'int32'|'bool'|'complex64'; +} + +export interface ValidationRun { + taskType: 'model'; + modelFormat: ModelFormat; + modelName: string; + functionName: ModelFunctionName; + inputs: {[key: string]: TensorDef}; + outputs: {[key: string]: TensorDef}; + async: boolean; +} diff --git a/src/util.ts b/src/util.ts new file mode 100644 index 0000000..94b2100 --- /dev/null +++ b/src/util.ts @@ -0,0 +1,44 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +import * as tf from '@tensorflow/tfjs-core'; + +// Maximum number of time before CPU tests don't execute during the next round. +export const LAST_RUN_CPU_CUTOFF_MS = 5000; + +export async function benchmark(benchmarkFn: () => tf.Tensor | tf.Tensor[]): + Promise { + // Use normal performance.now() timing even though query timers are enabled + // again because we want to account for more than just GPU time. + const start = performance.now(); + const result = benchmarkFn(); + await (result as tf.Tensor).data(); + return performance.now() - start; +} + +export async function asyncBenchmark( + asyncBenchmarkFn: () => + Promise| Promise): Promise { + const start = performance.now(); + const result = await asyncBenchmarkFn() as tf.Tensor[] | tf.Tensor; + + const outRes = Array.isArray(result) ? (result as tf.Tensor[])[0] : result; + await (outRes as tf.Tensor).data(); + (outRes as tf.Tensor).dispose(); + + return performance.now() - start; +} diff --git a/yarn.lock b/yarn.lock index cc35f9d..2a686af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15,6 +15,11 @@ optionalDependencies: rollup-plugin-visualizer "~1.1.1" +"@tensorflow/tfjs-layers@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-layers/-/tfjs-layers-1.2.1.tgz#78f28e90ee95ec206e69d5e2432b677e28c70c45" + integrity sha512-h3uFYRHzEGqtyzC4PLHcXGn2tbPcozk9H6yMEtdI1yLJZXBKTh4Yjs9B/yN9HRxp31pc2hCFrqU7ViNYAg/vYw== + "@types/estree@0.0.38": version "0.0.38" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.38.tgz#c1be40aa933723c608820a99a373a16d215a1ca2"