Add mobilenet benchmark

package.json

@@ -31,6 +31,7 @@
     "yalc": "~1.0.0-pre.21"
   },
   "dependencies": {
-    "@tensorflow/tfjs-core": "1.2.0"
+    "@tensorflow/tfjs-core": "1.2.0",
+    "@tensorflow/tfjs-layers": "^1.2.1"
   }
 }

src/benchmark_ops_test.ts

@@ -1,69 +1,86 @@
-/**
- * @license
- * Copyright 2019 Google LLC. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================================
- */
-
-import * as tf from '@tensorflow/tfjs-core';
-import * as tfwebgl2compute from './index';
-
-describe('Ops benchmarks', () => {
-  beforeAll(async () => await tfwebgl2compute.ready);
-
-  it('matMul', async () => {
-    const times = [];
-
-    const a = tf.randomNormal([500, 500]);
-    const b = tf.randomNormal([500, 500]);
-
-    let c = tf.matMul(a, b);
-    await c.data();
-
-    for (let i = 0; i < 100; i++) {
-      const start = performance.now();
-      c = tf.matMul(a, b);
-      await c.data();
-      times.push(performance.now() - start);
-    }
-
-    a.dispose();
-    b.dispose();
-    console.log(`MatMul: Average time ms: ${
-        times.reduce((a, b) => a + b, 0) / times.length}`);
-    console.log(`Min time ms: ${Math.min(...times)}`);
-  });
-
-  it('conv2d', async () => {
-    const times = [];
-
-    const a = tf.randomNormal<tf.Rank.R4>([1, 128, 128, 4]);
-    const b = tf.randomNormal<tf.Rank.R4>([25, 25, 4, 4]);
-
-    let c = tf.conv2d(a, b, 1, 'same');
-    await c.data();
-
-    for (let i = 0; i < 100; i++) {
-      const start = performance.now();
-      c = tf.conv2d(a, b, 1, 'same');
-      await c.data();
-      times.push(performance.now() - start);
-    }
-
-    a.dispose();
-    b.dispose();
-    console.log(`Conv2d: Average time ms: ${
-        times.reduce((a, b) => a + b, 0) / times.length}`);
-    console.log(`Min time ms: ${Math.min(...times)}`);
-  });
+/**
+ * @license
+ * Copyright 2019 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import * as tf from '@tensorflow/tfjs-core';
+import * as tfwebgl2compute from './index';
+import {MobileNetV1GPUBenchmark} from './mobilenet_benchmarks';
+import * as test_util from './test_util';
+
+describe('Ops benchmarks', () => {
+  beforeAll(async () => {
+    await tfwebgl2compute.ready;
+    jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000;
+  });
+
+    it('mobilenet_v1', async () => {
+    const sizes = [1];  // MobileNet version
+    const runs = 20;
+
+    const benchmark = new MobileNetV1GPUBenchmark();
+    await benchmark.loadModel();
+
+    await test_util.benchmarkAndLog(
+        'mobilenet_v1', size => benchmark.run(size), sizes,
+        size => `N=${size}_0_224`, runs);
+  });
+
+  it('matMul', async () => {
+    const times = [];
+
+    const a = tf.randomNormal([500, 500]);
+    const b = tf.randomNormal([500, 500]);
+
+    let c = tf.matMul(a, b);
+    await c.data();
+
+    for (let i = 0; i < 100; i++) {
+      const start = performance.now();
+      c = tf.matMul(a, b);
+      await c.data();
+      times.push(performance.now() - start);
+    }
+
+    a.dispose();
+    b.dispose();
+    console.log(`MatMul: Average time ms: ${
+        times.reduce((a, b) => a + b, 0) / times.length}`);
+    console.log(`Min time ms: ${Math.min(...times)}`);
+  });
+
+  it('conv2d', async () => {
+    const times = [];
+
+    const a = tf.randomNormal<tf.Rank.R4>([1, 128, 128, 4]);
+    const b = tf.randomNormal<tf.Rank.R4>([25, 25, 4, 4]);
+
+    let c = tf.conv2d(a, b, 1, 'same');
+    await c.data();
+
+    for (let i = 0; i < 100; i++) {
+      const start = performance.now();
+      c = tf.conv2d(a, b, 1, 'same');
+      await c.data();
+      times.push(performance.now() - start);
+    }
+
+    a.dispose();
+    b.dispose();
+    console.log(`Conv2d: Average time ms: ${
+        times.reduce((a, b) => a + b, 0) / times.length}`);
+    console.log(`Min time ms: ${Math.min(...times)}`);
+  });
 });

src/mobilenet_benchmarks.ts

@@ -0,0 +1,47 @@
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+import * as tfc from '@tensorflow/tfjs-core';
+import * as tfl from '@tensorflow/tfjs-layers';
+
+import {BenchmarkModelTest} from './types';
+import * as util from './util';
+
+const MOBILENET_MODEL_PATH =
+    // tslint:disable-next-line:max-line-length
+    'https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_1.0_224/model.json';
+
+export class MobileNetV1GPUBenchmark implements BenchmarkModelTest {
+  private model: tfl.LayersModel;
+
+  async loadModel() {
+    this.model = await tfl.loadLayersModel(MOBILENET_MODEL_PATH);
+  }
+
+  async run(size: number): Promise<number> {
+    tfc.setBackend('webgl');
+
+    const zeros = tfc.zeros([1, 224, 224, 3]);
+
+    const benchmark = () => this.model.predict(zeros);
+
+    const time = await util.benchmark(benchmark);
+
+    zeros.dispose();
+
+    return time;
+  }
+}

src/setup_test.ts

@@ -126,4 +126,4 @@ env.specFilter = spec => {
 };
 
 // Import and run all the tests from core.
-import '@tensorflow/tfjs-core/dist/tests';
+//import '@tensorflow/tfjs-core/dist/tests';

src/test_util.ts

@@ -0,0 +1,51 @@
+/**
+ * @license
+ * Copyright 2019 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+//import * as firebase from './firebase';
+import {BenchmarkLog} from './types';
+
+function nextTick(): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve));
+}
+
+// tslint:disable-next-line:no-any
+export async function benchmarkAndLog<T extends any>(
+    name: string, benchmark: (size: T) => Promise<number>, sizes: T[],
+    sizeToParams: (size: T) => string, runCount = 100,
+    warmupRunCount = 1): Promise<void> {
+  const logs: BenchmarkLog[] = [];
+
+  for (let i = 0; i < sizes.length; i++) {
+    const size = sizes[i];
+    let averageTimeMs = 0;
+    let result;
+
+    for (let j = 0; j < warmupRunCount; j++) {
+      result = await benchmark(size);
+    }
+
+    for (let j = 0; j < runCount; j++) {
+      result = await benchmark(size);
+      averageTimeMs += result / runCount;
+      await nextTick();
+    }
+    const benchmarkLog:
+        BenchmarkLog = {params: sizeToParams(size), averageTimeMs};
+    logs.push(benchmarkLog);
+  }
+  //await firebase.logBenchmarkRun(name, logs);
+}