avoid gcp provisioning for google workspace

docs/sources/google-workspace/README.md

@@ -108,16 +108,32 @@ While not recommended, it is possible to set up Google API clients without Terra
 
 Then follow the steps in the next section to create the keys for the Oauth Clients.
 
+If your organization's policies don't allow Terraform to manage some or all of these GCP resources, you can still use our Terraform modules for the rest of your deployment and disable the parts you must do manually via `google_workspace_connector_settings` in your `terraform.tfvars`:
+
+```hcl
+google_workspace_connector_settings = {
+  enable_apis                = false
+  provision_service_accounts = false
+  provision_keys             = false
+}
+```
+
+When any of these are `false`, Terraform will skip creating the corresponding resources and instead emit TODO files (or `todos_1` outputs, if configured) with instructions to complete those steps outside of Terraform.
+
 NOTE: if you are creating connections to multiple Google Workspace™ sources, you can use a single OAuth client and share it between all the proxy instances. You just need to authorize the entire superset of Oauth scopes required by those connnections for the OAuth Client via the Google Workspace™ Admin console.
 
 ### Provisioning API Keys without Terraform
 
 If your organization's policies don't allow GCP service account keys to be managed via Terraform (or you lack the perms to do so), you can still use our Terraform modules to create the clients, and just add the following to your `terraform.tfvars` to disable provisioning of the keys:
 
 ```hcl
-google_workspace_provision_keys = false
+google_workspace_connector_settings = {
+  provision_keys = false
+}
 ```
 
+The deprecated top-level variable `google_workspace_provision_keys` is still supported, but the map form above is preferred.
+
 Then you can create the keys manually, and store them in your secrets manager of choice.
 
 For each API client you need to:

infra/examples-dev/aws/google-workspace-variables.tf

@@ -80,6 +80,6 @@ locals {
 
 variable "google_workspace_connector_settings" {
   type        = map(any)
-  description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
+  description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
   default     = {}
 }

infra/examples-dev/gcp/google-workspace-variables.tf

@@ -80,6 +80,6 @@ locals {
 
 variable "google_workspace_connector_settings" {
   type        = map(any)
-  description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
+  description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
   default     = {}
 }

infra/modules/google-workspace-dwd-connection/main.tf

@@ -19,19 +19,29 @@ locals {
   # TODO: md5 here is 32 chars of hex, so some risk of collision by truncating
   sa_account_id = length(local.padded_id) < 31 ? local.padded_id : substr(md5(local.padded_id), 0, 30)
 
-  instance_id = coalesce(var.instance_id, var.display_name)
+  instance_id        = coalesce(var.instance_id, var.display_name)
+  expected_sa_email  = "${local.sa_account_id}@${var.project_id}.iam.gserviceaccount.com"
+  oauth_client_id    = var.provision_service_account ? google_service_account.connector_sa[0].unique_id : "REPLACE_WITH_NUMERIC_CLIENT_ID_AFTER_CREATING_SERVICE_ACCOUNT"
+  service_account_email_for_todo = var.provision_service_account ? google_service_account.connector_sa[0].email : local.expected_sa_email
 }
 
 # service account to personify connector
+moved {
+  from = google_service_account.connector_sa
+  to   = google_service_account.connector_sa[0]
+}
+
 resource "google_service_account" "connector_sa" {
+  count = var.provision_service_account ? 1 : 0
+
   project      = var.project_id
   account_id   = local.sa_account_id
   display_name = var.display_name
   description  = var.description
 }
 
 resource "google_project_service" "apis_needed" {
-  for_each = toset(var.apis_consumed)
+  for_each = var.enable_apis ? toset(var.apis_consumed) : toset([])
 
   service                    = each.key
   project                    = var.project_id
@@ -88,41 +98,48 @@ connection to will fail)
      Account to Use for Connection' setting when they create the connection.
 
   8. Optionally, you may also set the email address of the account you created the value of
-     `google_workspace_example_user` in your `terraform.tfvars` file. This will cause the example
+     `google_workspace_connector_settings` (eg, `example_user`) in your `terraform.tfvars` file. This will cause the example
      API invocations generated by the terraform modules to prefill this value as the account to
      impersonate on those requests. This will allow you to validate the permissions of the account,
      as well as the ability of the proxy connection to impersonate it.
 EOT
 
+  manual_sa_todo_note = var.provision_service_account ? "" : <<-EOT
+
+     NOTE: Terraform did not provision this service account. After you create it manually, replace
+     the placeholder client ID above with the numeric ID shown in the GCP console for
+     `${local.expected_sa_email}`.
+EOT
+
   todo_content = <<EOT
 Complete the following steps via the Google Workspace Admin console:
   1. Visit https://admin.google.com/ and navigate to "Security" --> "Access and Data Control" -->
      "API Controls", then find "Manage Domain Wide Delegation". Click "Add new".
 
-  2. Copy and paste client ID `${google_service_account.connector_sa.unique_id}` into the
+  2. Copy and paste client ID `${local.oauth_client_id}` into the
      "Client ID" input in the popup. (this is the unique ID of the GCP service account with
-     email `${google_service_account.connector_sa.email}`; you can (and should) verify its identity
-     via the GCP console, with the project `${google_service_account.connector_sa.project}`, under:
+     email `${local.service_account_email_for_todo}`; you can (and should) verify its identity
+     via the GCP console, with the project `${var.project_id}`, under:
 
-     ["IAM & Admin" --> "Service Accounts"](https://console.cloud.google.com/iam-admin/serviceaccounts?project=${google_service_account.connector_sa.project}&supportedpurview=project)
+     ["IAM & Admin" --> "Service Accounts"](https://console.cloud.google.com/iam-admin/serviceaccounts?project=${var.project_id}&supportedpurview=project)
 
      This ensures you are granting domain-wide delegation to the correct service account, and
      mitigates the risk that these instructions were forged by a malicious actor.
-
+${local.manual_sa_todo_note}
      Via the GCP console, you can also verify all extant keys for the service account, to ensure
      that there is exactly one, which should be held by the proxy.  GCP provides log of key usage,
      creation, revocation, etc, which you can monitor to ensure that the key is being used only by
      the proxy, only for the data access you expect. If you ever suspect compromise, you may revoke
-     the key from the GCP console at any time (NOTE: that proxy connection will be broken until your
-     Terraform configuration is re-applied, to provision a new key).
+     the key from the GCP console at any time (NOTE: that proxy connection will be broken until a new
+     key is provisioned and stored in your secrets manager).
 
   3. Copy and paste the following OAuth 2.0 scope string into the "Scopes" input:
 ```
 ${join(",", var.oauth_scopes_needed)}
 ```
 
    4. Authorize it. With this, your psoxy instance should be able to authenticate with Google as
-      the GCP Service Account `${google_service_account.connector_sa.email}` and request data from
+      the GCP Service Account `${local.service_account_email_for_todo}` and request data from
       Google as authorized by the OAuth scopes you granted.
 ${local.google_workspace_admin_account_required ? local.google_workspace_service_account_setup : ""}
 EOT

infra/modules/google-workspace-dwd-connection/output.tf

@@ -3,15 +3,16 @@ output "instance_id" {
 }
 
 output "service_account_id" {
-  value = google_service_account.connector_sa.id
+  value = var.provision_service_account ? google_service_account.connector_sa[0].id : "projects/${var.project_id}/serviceAccounts/${local.expected_sa_email}"
 }
 
 output "service_account_email" {
-  value = google_service_account.connector_sa.email
+  value = var.provision_service_account ? google_service_account.connector_sa[0].email : local.expected_sa_email
 }
 
 output "service_account_numeric_id" {
-  value = google_service_account.connector_sa.unique_id
+  value       = var.provision_service_account ? google_service_account.connector_sa[0].unique_id : null
+  description = "OAuth client ID for domain-wide delegation; null if the service account is not provisioned by Terraform"
 }
 
 output "next_todo_step" {

infra/modules/google-workspace-dwd-connection/variables.tf

@@ -36,6 +36,18 @@ variable "oauth_scopes_needed" {
   default     = []
 }
 
+variable "provision_service_account" {
+  type        = bool
+  description = "whether to provision the GCP service account (OAuth client) via Terraform. If false, you must create it manually."
+  default     = true
+}
+
+variable "enable_apis" {
+  type        = bool
+  description = "whether to enable required GCP APIs via Terraform. If false, you must enable them manually."
+  default     = true
+}
+
 variable "todos_as_local_files" {
   type        = bool
   description = "whether to render TODOs as flat files"
@@ -47,4 +59,3 @@ variable "todo_step" {
   description = "of all todos, where does this one logically fall in sequence"
   default     = 1
 }
-

infra/modules/worklytics-connector-specs/google-workspace.tf

@@ -1,11 +1,11 @@
 locals {
 
   google_workspace_example_user = try(
-    coalesce(var.google_workspace_connector_settings["google_workspace_example_user"]),
+    coalesce(var.google_workspace_connector_settings["example_user"]),
     coalesce(var.google_workspace_example_user, "REPLACE_WITH_EXAMPLE_USER@YOUR_COMPANY.COM")
   )
   google_workspace_example_admin = try(
-    coalesce(var.google_workspace_connector_settings["google_workspace_example_admin"]),
+    coalesce(var.google_workspace_connector_settings["example_admin"]),
     coalesce(var.google_workspace_example_admin, local.google_workspace_example_user, "REPLACE_WITH_EXAMPLE_ADMIN@YOUR_COMPANY.COM")
   )
   google_workspace_sources = {

infra/modules/worklytics-connector-specs/variables.tf

@@ -237,6 +237,6 @@ variable "msft_365_connector_settings" {
 
 variable "google_workspace_connector_settings" {
   type        = map(any)
-  description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
+  description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
   default     = {}
 }

infra/modules/worklytics-connectors-google-workspace/gcp-api-enable-todo.tftpl

@@ -0,0 +1,16 @@
+In the GCP console for `${gcp_project_id}` (or via `gcloud`), enable the following APIs required by the `${connector_id}` connector:
+
+%{ for api in apis_consumed ~}
+- `${api}`
+%{ endfor ~}
+
+Via the GCP console: navigate to "APIs & Services" --> "Library", search for each API above, and click "Enable".
+
+Via gcloud (one command per API):
+%{ for api in apis_consumed ~}
+`gcloud services enable ${api} --project=${gcp_project_id}`
+%{ endfor ~}
+
+See the page below for more information on provisioning Google Workspace connectors without Terraform:
+
+https://docs.worklytics.co/psoxy/sources/google-workspace#provisioning-api-clients-without-terraform

infra/modules/worklytics-connectors-google-workspace/gcp-sa-create-todo.tftpl

@@ -0,0 +1,18 @@
+In the GCP console for `${gcp_project_id}` (or via `gcloud`), create a service account to use as the OAuth client for the `${connector_id}` connector:
+
+- **Account ID**: `${service_account_id}`
+- **Display name**: `${display_name}`
+- **Description**: `${description}`
+
+The service account email should be `${expected_service_account_email}`.
+
+Via the GCP console: navigate to "IAM & Admin" --> "Service Accounts" --> "Create Service Account", and use the values above.
+
+Via gcloud:
+`gcloud iam service-accounts create ${service_account_id} --display-name="${display_name}" --description="${description}" --project=${gcp_project_id}`
+
+After creating the service account, note its numeric **Client ID** (unique ID) from the service account details page. You will need it when completing domain-wide delegation setup.
+
+See the page below for more information on provisioning Google Workspace connectors without Terraform:
+
+https://docs.worklytics.co/psoxy/sources/google-workspace#provisioning-api-clients-without-terraform

infra/modules/worklytics-connectors-google-workspace/main.tf

@@ -1,6 +1,18 @@
 locals {
-  provision_gcp_sa_keys    = try(var.google_workspace_connector_settings["google_workspace_provision_keys"], var.provision_gcp_sa_keys)
-  gcp_sa_key_rotation_days = try(var.google_workspace_connector_settings["google_workspace_key_rotation_days"], var.gcp_sa_key_rotation_days)
+  provision_service_accounts = try(var.google_workspace_connector_settings["provision_service_accounts"], true)
+  enable_apis                = try(var.google_workspace_connector_settings["enable_apis"], true)
+  provision_gcp_sa_keys = (
+    local.provision_service_accounts
+    ? try(var.google_workspace_connector_settings["provision_keys"], var.provision_gcp_sa_keys)
+    : false
+  )
+  gcp_sa_key_rotation_days = try(var.google_workspace_connector_settings["key_rotation_days"], var.gcp_sa_key_rotation_days)
+
+  manual_steps_before_dwd = (local.enable_apis ? 0 : 1) + (local.provision_service_accounts ? 0 : 1)
+  dwd_todo_step           = var.todo_step + local.manual_steps_before_dwd
+  api_todo_step           = var.todo_step
+  sa_todo_step              = var.todo_step + (local.enable_apis ? 0 : 1)
+  key_todo_step             = local.dwd_todo_step + 1
 }
 terraform {
   required_version = "~> 1.7"
@@ -46,24 +58,65 @@ module "google_workspace_connection" {
   description                  = "Google API OAuth Client for ${each.value.display_name}"
   apis_consumed                = each.value.apis_consumed
   oauth_scopes_needed          = each.value.oauth_scopes_needed
+  provision_service_account    = local.provision_service_accounts
+  enable_apis                  = local.enable_apis
   todos_as_local_files         = var.todos_as_local_files
-  todo_step                    = var.todo_step
+  todo_step                    = local.dwd_todo_step
 }
 
 locals {
 
+  api_enable_todos = {
+    for id, connection in module.google_workspace_connection :
+    id => templatefile("${path.module}/gcp-api-enable-todo.tftpl", {
+      gcp_project_id : var.gcp_project_id
+      connector_id : id
+      apis_consumed : module.worklytics_connector_specs.enabled_google_workspace_connectors[id].apis_consumed
+    })
+  }
+
+  sa_creation_todos = {
+    for id, connection in module.google_workspace_connection :
+    id => templatefile("${path.module}/gcp-sa-create-todo.tftpl", {
+      gcp_project_id : var.gcp_project_id
+      connector_id : id
+      service_account_id : "${local.environment_id_prefix}${substr(id, 0, 30 - length(local.environment_id_prefix))}"
+      display_name : "Psoxy Connector - ${local.environment_id_display_name_qualifier}${module.worklytics_connector_specs.enabled_google_workspace_connectors[id].display_name}"
+      description : "Google API OAuth Client for ${module.worklytics_connector_specs.enabled_google_workspace_connectors[id].display_name}"
+      expected_service_account_email : connection.service_account_email
+    })
+  }
+
   key_creation_todos = {
     for id, connection in module.google_workspace_connection :
     id => templatefile("${path.module}/gcp-sa-key-create-todo.tftpl", { gcp_project_id : var.gcp_project_id, gcp_service_account : connection.service_account_email, secret_prefix : connection.instance_id })
   }
 
-  todos = [for id, connection in module.google_workspace_connection :
-    local.provision_gcp_sa_keys ? connection.todo : "${local.key_creation_todos[id]}\n${connection.todo}"
-  ]
+  connector_todos = {
+    for id, connection in module.google_workspace_connection :
+    id => join("\n\n", [for part in [
+      local.enable_apis ? null : local.api_enable_todos[id],
+      local.provision_service_accounts ? null : local.sa_creation_todos[id],
+      connection.todo,
+      local.provision_gcp_sa_keys ? null : local.key_creation_todos[id],
+    ] : part if part != null])
+  }
+
+  todos = [for id, connection in module.google_workspace_connection : local.connector_todos[id]]
 
-  current_todo_step = try(max(values(module.google_workspace_connection)[*].next_todo_step...), var.todo_step)
+  current_todo_step = try(max(values(module.google_workspace_connection)[*].next_todo_step...), local.dwd_todo_step)
   next_todo_step    = local.provision_gcp_sa_keys ? local.current_todo_step : local.current_todo_step + 1
 
+  connectors_needing_manual_api_enablement = local.enable_apis ? {} : {
+    for k, v in module.worklytics_connector_specs.enabled_google_workspace_connectors :
+    k => v
+  }
+
+  connectors_needing_manual_sa_creation = local.provision_service_accounts ? {} : {
+    for k, v in module.worklytics_connector_specs.enabled_google_workspace_connectors :
+    k => v
+  }
+
   service_accounts_tf_managed_keys = local.provision_gcp_sa_keys ? {
     for k, v in module.worklytics_connector_specs.enabled_google_workspace_connectors :
     k => module.google_workspace_connection[k].service_account_id
@@ -75,10 +128,24 @@ locals {
   }
 }
 
+resource "local_file" "todo_gcp_api_enablement" {
+  for_each = var.todos_as_local_files ? local.connectors_needing_manual_api_enablement : {}
+
+  filename = "TODO ${local.api_todo_step} - Enable APIs for ${each.key}.md"
+  content  = local.api_enable_todos[each.key]
+}
+
+resource "local_file" "todo_gcp_sa_creation" {
+  for_each = var.todos_as_local_files ? local.connectors_needing_manual_sa_creation : {}
+
+  filename = "TODO ${local.sa_todo_step} - Create Service Account for ${each.key}.md"
+  content  = local.sa_creation_todos[each.key]
+}
+
 resource "local_file" "todo_gcp_sa_key_creation" {
   for_each = var.todos_as_local_files ? local.service_accounts_user_managed_keys : {}
 
-  filename = "TODO ${local.current_todo_step} - Create Key for ${each.key}.md"
+  filename = "TODO ${local.key_todo_step} - Create Key for ${each.key}.md"
   content  = local.key_creation_todos[each.key]
 }
 

infra/modules/worklytics-connectors-google-workspace/variables.tf

@@ -50,7 +50,7 @@ variable "google_workspace_example_admin" {
 
 variable "provision_gcp_sa_keys" {
   type        = bool
-  description = "whether to provision key for each connector's GCP Service Account (OAuth Client). If false, you must create the key manually and provide it."
+  description = "[DEPRECATED - use google_workspace_connector_settings map instead] whether to provision key for each connector's GCP Service Account (OAuth Client). If false, you must create the key manually and provide it. Ignored if service accounts are not provisioned by Terraform."
   default     = true
 }
 
@@ -80,6 +80,6 @@ variable "todo_step" {
 
 variable "google_workspace_connector_settings" {
   type        = map(any)
-  description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
+  description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables."
   default     = {}
 }