Setup release e2e infra. (#3632)

Author: lina128

cloudbuild.yml

@@ -129,7 +129,7 @@ steps:
   id: 'e2e'
   args: ['./scripts/run-build.sh', 'e2e']
   waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
+  env: ['NIGHTLY=$_NIGHTLY','RELEASE=$_RELEASE']
 
 # Inference API
 - name: 'gcr.io/cloud-builders/gcloud'

e2e/cloudbuild.yml

@@ -6,6 +6,7 @@ steps:
   entrypoint: 'bash'
   id: 'verdaccio'
   args: ['./scripts/start-verdaccio.sh']
+  env: ['RELEASE=$_RELEASE']
 
 # Install packages.
 - name: 'node:10'

e2e/package.json

@@ -60,7 +60,8 @@
     "lint": "tslint -p . -t verbose",
     "run-browserstack": "karma start --browserstack",
     "test": "./scripts/test.sh",
-    "test-ci": "./scripts/test-ci.sh"
+    "test-ci": "./scripts/test-ci.sh",
+    "update-dependency": "ts-node ./scripts/update-dependency.ts"
   },
   "license": "Apache-2.0",
   "engines": {

e2e/scripts/publish-monorepo-ci.sh renamed to e2e/scripts/publish-tfjs-ci.sh

@@ -39,7 +39,6 @@ set -x
 
 # Go to root
 cd ../../
-root_path=$PWD
 
 # Yarn in the top-level
 yarn
@@ -79,3 +78,8 @@ do
 
   cd ..
 done
+
+# Update e2e's package.json's all tfjs related packages to locally published
+# version.
+cd e2e
+yarn update-dependency --version=$RELEASE_VERSION

e2e/scripts/start-verdaccio.sh

@@ -77,10 +77,9 @@ if [[ "$RELEASE" = true ]]; then
   # Start the local NPM registry
   startLocalRegistry "$e2e_root_path"/scripts/verdaccio.yaml
 
-  # Publish the monorepo
-  "$e2e_root_path"/scripts/publish-monorepo-ci.sh
-
-  # Todo(linazhao): Revise package.json to use the published version.
+  # Publish the monorepo and update package.json tfjs dependency to the
+  # published version.
+  "$e2e_root_path"/scripts/publish-tfjs-ci.sh
 
   # Cleanup
   cleanup

e2e/scripts/update-dependency.ts

@@ -0,0 +1,66 @@
+// Copyright 2020 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+
+// Run this script from the e2e root directory:
+// yarn update-dependency --version=VERSION
+// Where VERSION is the pinned version to update to for tfjs package
+// dependencies.
+
+import * as argparse from 'argparse';
+import * as fs from 'fs';
+
+import {updateTFJSDependencyVersions} from '../../scripts/release-util';
+
+const parser = new argparse.ArgumentParser();
+parser.addArgument('--version', {
+  required: true,
+  help: 'Use the git protocal rather than the http protocol when cloning repos.'
+});
+
+const DEPENDENCY_LIST = [
+  'tfjs-core', 'tfjs-converter', 'tfjs-layers', 'tfjs-backend-cpu',
+  'tfjs-backend-webgl', 'tfjs-data', 'tfjs', 'tfjs-node'
+];
+
+function main() {
+  const args = parser.parseArgs();
+  const latestVersion = args.version;
+
+  if (!latestVersion) {
+    console.log('Please pass a valid version.');
+    process.exit(1);
+  }
+
+  const packageJsonFile = './package.json';
+  if (!fs.existsSync(packageJsonFile)) {
+    console.log(
+        packageJsonFile,
+        'does not exist. Please call this script from the e2e root directory.');
+    process.exit(1);
+  }
+
+  // Update the version.
+  let pkg = fs.readFileSync(packageJsonFile, 'utf8');
+  const parsedPkg = JSON.parse(pkg);
+
+  pkg = updateTFJSDependencyVersions(
+      DEPENDENCY_LIST, pkg, parsedPkg, latestVersion);
+
+  fs.writeFileSync(packageJsonFile, pkg);
+
+  process.exit(0);
+}
+
+main();

e2e/scripts/verdaccio.yaml

@@ -1,5 +1,9 @@
-# path to a directory with all packages
+# Path to a directory with all packages.
 storage: ./storage
+
+# Maximum body size for a JSON document.
+max_body_size: 1000mb
+
 auth:
   htpasswd:
     file: ./htpasswd

scripts/release-tfjs.ts

@@ -101,6 +101,9 @@ async function main() {
       }
     }
   }
+  // Insert an empty release file at the root level, so that CI can use it to
+  // set RELEASE flag.
+  fs.writeFileSync('release', '');
 
   // Use dev prefix to avoid branch being locked.
   const devBranchName = `dev_${releaseBranch}`;

scripts/release-util.ts

@@ -1,5 +1,23 @@
+#!/usr/bin/env node
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
 import chalk from 'chalk';
-import * as mkdirp from 'mkdirp';
+import mkdirp from 'mkdirp';
 import * as readline from 'readline';
 import * as shell from 'shelljs';
 

scripts/run-build.sh

@@ -14,10 +14,31 @@
 # limitations under the License.
 # =============================================================================
 
+# Echo every command being executed
+set -x
+
+# Exit the script on any command with non 0 return code
 set -e
 
+# Use the top level empty release file as a RELEASE flag.
+RELEASE=false
+if [[ -f "release" ]]; then
+  $RELEASE=true
+fi
+
 DIR=$1
-if [[ -f "$DIR/run-ci" || "$NIGHTLY" = true || $DIR == "e2e" ]]; then
-  gcloud builds submit . --config=$DIR/cloudbuild.yml \
-    --substitutions _NIGHTLY=$NIGHTLY
+if [[ $RELEASE = true ]]; then
+  # Release flow: Only run e2e release test, ignore unit tests in all other
+  # directories.
+  if [[ $DIR = "e2e" ]]; then
+    gcloud build submit . --config=$DIR/cloudbuild.yml \
+      --substitutions _RELEASE=$RELEASE
+  fi
+else
+  # Regular flow: Only run changed packages plus e2e regular test.
+  # Nightly flow: Run everything.
+  if [[ -f "$DIR/run-ci" || $DIR == "e2e" || $NIGHTLY = true ]]; then
+    gcloud builds submit . --config=$DIR/cloudbuild.yml \
+      --substitutions _NIGHTLY=$NIGHTLY
+  fi
 fi