Add warning when emulating extensions that use unemulated APIs (#4226)

* starting to check APIs

* merging

* more progress

* more progress

* Adding check for unemulated APIs

* reverting shrinkwrap changes

* reverting shrinkwrap changes

* clean up unused import

* add enable API console link

* more pr fixes

* adding renamed files

* one last round of pr clean up

* clean up unused marked

src/emulator/extensions/validation.ts

@@ -0,0 +1,44 @@
+import * as planner from "../../deploy/extensions/planner";
+import { check } from "../../ensureApiEnabled";
+
+const EMULATED_APIS = [
+  "storage-component.googleapis.com",
+  "firestore.googleapis.com",
+  "pubsub.googleapis.com",
+  "identitytoolkit.googleapis.com",
+  // TODO: Is there a RTDB API we need to add here? I couldn't find one.
+];
+
+type APIInfo = {
+  apiName: string;
+  instanceIds: string[];
+  enabled: boolean;
+};
+/**
+ * getUnemulatedAPIs checks a list of InstanceSpecs for APIs that are not emulated.
+ * It returns a map of API name to list of instanceIds that use that API.
+ */
+export async function getUnemulatedAPIs(
+  projectId: string,
+  instances: planner.InstanceSpec[]
+): Promise<APIInfo[]> {
+  const unemulatedAPIs: Record<string, APIInfo> = {};
+  for (const i of instances) {
+    const extensionVersion = await planner.getExtensionVersion(i);
+    for (const api of extensionVersion.spec.apis ?? []) {
+      if (!EMULATED_APIS.includes(api.apiName)) {
+        if (unemulatedAPIs[api.apiName]) {
+          unemulatedAPIs[api.apiName].instanceIds.push(i.instanceId);
+        } else {
+          const enabled = await check(projectId, api.apiName, "extensions", true);
+          unemulatedAPIs[api.apiName] = {
+            apiName: api.apiName,
+            instanceIds: [i.instanceId],
+            enabled,
+          };
+        }
+      }
+    }
+  }
+  return Object.values(unemulatedAPIs);
+}

src/emulator/extensionsEmulator.ts

@@ -1,6 +1,8 @@
 import * as fs from "fs-extra";
 import * as os from "os";
 import * as path from "path";
+import * as clc from "cli-color";
+import Table = require("cli-table");
 import { spawnSync } from "child_process";
 
 import * as planner from "../deploy/extensions/planner";
@@ -10,6 +12,10 @@ import { downloadExtensionVersion } from "./download";
 import { EmulatableBackend } from "./functionsEmulator";
 import { getExtensionFunctionInfo } from "../extensions/emulator/optionsHelper";
 import { EmulatorLogger } from "./emulatorLogger";
+import { Emulators } from "./types";
+import { getUnemulatedAPIs } from "./extensions/validation";
+import { enableApiURI } from "../ensureApiEnabled";
+import { shortenUrl } from "../shortenUrl";
 
 export interface ExtensionEmulatorArgs {
   projectId: string;
@@ -24,6 +30,7 @@ export interface ExtensionEmulatorArgs {
 export class ExtensionsEmulator {
   private want: planner.InstanceSpec[] = [];
   private args: ExtensionEmulatorArgs;
+  private logger = EmulatorLogger.forEmulator(Emulators.EXTENSIONS);
 
   constructor(args: ExtensionEmulatorArgs) {
     this.args = args;
@@ -120,10 +127,11 @@ export class ExtensionsEmulator {
    *  getEmulatableBackends reads firebase.json & .env files for a list of extension instances to emulate,
    *  downloads & builds the necessary source code (if it hasn't previously been cached),
    *  then builds returns a list of emulatableBackends
-   *  @returns A list of emulatableBackends, one for each extension instance to be emulated
+   *  @return A list of emulatableBackends, one for each extension instance to be emulated
    */
   public async getExtensionBackends(): Promise<EmulatableBackend[]> {
     await this.readManifest();
+    await this.checkAndWarnAPIs(this.want);
     return Promise.all(
       this.want.map((i: planner.InstanceSpec) => {
         return this.toEmulatableBackend(i);
@@ -169,4 +177,39 @@ export class ExtensionsEmulator {
       STORAGE_BUCKET: `${projectId}.appspot.com`,
     };
   }
+
+  private async checkAndWarnAPIs(instances: planner.InstanceSpec[]): Promise<void> {
+    const apisToWarn = await getUnemulatedAPIs(this.args.projectId, instances);
+    if (apisToWarn.length) {
+      const table = new Table({
+        head: [
+          "API Name",
+          "Instances using this API",
+          `Enabled on ${this.args.projectId}`,
+          `Enable this API`,
+        ],
+        style: { head: ["yellow"] },
+      });
+      for (const apiToWarn of apisToWarn) {
+        // We use a shortened link here instead of a alias because cli-table behaves poorly with aliased links
+        const enablementUri = await shortenUrl(
+          enableApiURI(this.args.projectId, apiToWarn.apiName)
+        );
+        table.push([
+          apiToWarn.apiName,
+          apiToWarn.instanceIds,
+          apiToWarn.enabled ? "Yes" : "No",
+          apiToWarn.enabled ? "" : clc.bold.underline(enablementUri),
+        ]);
+      }
+
+      this.logger.logLabeled(
+        "WARN",
+        "Extensions",
+        `The following Extensions make calls to Google Cloud APIs that do not have Emulators. ` +
+          `These calls will go to production Google Cloud APIs which may have real effects on ${this.args.projectId}.\n` +
+          table.toString()
+      );
+    }
+  }
 }

src/ensureApiEnabled.ts

@@ -132,3 +132,15 @@ export async function ensure(
   }
   return enableApiWithRetries(projectId, apiName, prefix, silent);
 }
+
+/**
+ * Returns a link to enable an API on a project in Cloud console. This can be used instead of ensure
+ * in contexts where automatically enabling APIs is not desirable (ie emulator commands).
+ *
+ * @param projectId The project to generate an API enablement link for
+ * @param apiName  The name of the API e.g. `someapi.googleapis.com`.
+ * @return A link to Cloud console to enable the API
+ */
+export function enableApiURI(projectId: string, apiName: string): string {
+  return `https://console.cloud.google.com/apis/library/${apiName}?project=${projectId}`;
+}

src/test/emulators/extensions/validation.spec.ts

@@ -0,0 +1,66 @@
+import { expect } from "chai";
+import * as sinon from "sinon";
+
+import * as utils from "../../../emulator/extensions/validation";
+import * as ensureApiEnabled from "../../../ensureApiEnabled";
+import { InstanceSpec } from "../../../deploy/extensions/planner";
+
+function getTestInstanceSpecWithAPI(instanceId: string, apiName: string): InstanceSpec {
+  return {
+    instanceId,
+    params: {},
+    extensionVersion: {
+      name: "publishers/test/extensions/test/versions/0.1.0",
+      ref: "test/[email protected]",
+      state: "PUBLISHED",
+      sourceDownloadUri: "test.com",
+      hash: "abc123",
+      spec: {
+        name: "test",
+        version: "0.1.0",
+        sourceUrl: "test.com",
+        resources: [],
+        params: [],
+        apis: [{ apiName, reason: "because" }],
+      },
+    },
+  };
+}
+
+describe("ExtensionsEmulator validation utils", () => {
+  describe(`${utils.getUnemulatedAPIs.name}`, () => {
+    const testProjectId = "test-project";
+    const testAPI = "test.googleapis.com";
+    const sandbox = sinon.createSandbox();
+
+    beforeEach(() => {
+      const checkStub = sandbox.stub(ensureApiEnabled, "check");
+      checkStub.withArgs(testProjectId, testAPI, "extensions", true).resolves(true);
+      checkStub.throws("Unexpected API checked in test");
+    });
+
+    afterEach(() => {
+      sandbox.restore();
+    });
+
+    it("should check only unemulated APIs", async () => {
+      const instanceIdWithUnemulatedAPI = "unemulated";
+      const instanceId2WithUnemulatedAPI = "unemulated2";
+      const instanceIdWithEmulatedAPI = "emulated";
+
+      const result = await utils.getUnemulatedAPIs(testProjectId, [
+        getTestInstanceSpecWithAPI(instanceIdWithEmulatedAPI, "firestore.googleapis.com"),
+        getTestInstanceSpecWithAPI(instanceIdWithUnemulatedAPI, testAPI),
+        getTestInstanceSpecWithAPI(instanceId2WithUnemulatedAPI, testAPI),
+      ]);
+
+      expect(result).to.deep.equal([
+        {
+          apiName: testAPI,
+          instanceIds: [instanceIdWithUnemulatedAPI, instanceId2WithUnemulatedAPI],
+          enabled: true,
+        },
+      ]);
+    });
+  });
+});