diff --git a/docs/sources/google-workspace/README.md b/docs/sources/google-workspace/README.md index 90251fe2e1..90c0c02927 100644 --- a/docs/sources/google-workspace/README.md +++ b/docs/sources/google-workspace/README.md @@ -108,6 +108,18 @@ While not recommended, it is possible to set up Google API clients without Terra Then follow the steps in the next section to create the keys for the Oauth Clients. +If your organization's policies don't allow Terraform to manage some or all of these GCP resources, you can still use our Terraform modules for the rest of your deployment and disable the parts you must do manually via `google_workspace_connector_settings` in your `terraform.tfvars`: + +```hcl +google_workspace_connector_settings = { + enable_apis = false + provision_service_accounts = false + provision_keys = false +} +``` + +When any of these are `false`, Terraform will skip creating the corresponding resources and instead emit TODO files (or `todos_1` outputs, if configured) with instructions to complete those steps outside of Terraform. + NOTE: if you are creating connections to multiple Google Workspace™ sources, you can use a single OAuth client and share it between all the proxy instances. You just need to authorize the entire superset of Oauth scopes required by those connnections for the OAuth Client via the Google Workspace™ Admin console. ### Provisioning API Keys without Terraform @@ -115,9 +127,13 @@ NOTE: if you are creating connections to multiple Google Workspace™ source If your organization's policies don't allow GCP service account keys to be managed via Terraform (or you lack the perms to do so), you can still use our Terraform modules to create the clients, and just add the following to your `terraform.tfvars` to disable provisioning of the keys: ```hcl -google_workspace_provision_keys = false +google_workspace_connector_settings = { + provision_keys = false +} ``` +The deprecated top-level variable `google_workspace_provision_keys` is still supported, but the map form above is preferred. + Then you can create the keys manually, and store them in your secrets manager of choice. For each API client you need to: diff --git a/infra/examples-dev/aws/google-workspace-variables.tf b/infra/examples-dev/aws/google-workspace-variables.tf index 8ceca1f169..cf63f376b4 100644 --- a/infra/examples-dev/aws/google-workspace-variables.tf +++ b/infra/examples-dev/aws/google-workspace-variables.tf @@ -80,6 +80,6 @@ locals { variable "google_workspace_connector_settings" { type = map(any) - description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." + description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." default = {} } diff --git a/infra/examples-dev/gcp/google-workspace-variables.tf b/infra/examples-dev/gcp/google-workspace-variables.tf index 8ceca1f169..cf63f376b4 100644 --- a/infra/examples-dev/gcp/google-workspace-variables.tf +++ b/infra/examples-dev/gcp/google-workspace-variables.tf @@ -80,6 +80,6 @@ locals { variable "google_workspace_connector_settings" { type = map(any) - description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." + description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." default = {} } diff --git a/infra/modules/google-workspace-dwd-connection/main.tf b/infra/modules/google-workspace-dwd-connection/main.tf index d82ea9364d..b229359f10 100644 --- a/infra/modules/google-workspace-dwd-connection/main.tf +++ b/infra/modules/google-workspace-dwd-connection/main.tf @@ -19,11 +19,21 @@ locals { # TODO: md5 here is 32 chars of hex, so some risk of collision by truncating sa_account_id = length(local.padded_id) < 31 ? local.padded_id : substr(md5(local.padded_id), 0, 30) - instance_id = coalesce(var.instance_id, var.display_name) + instance_id = coalesce(var.instance_id, var.display_name) + expected_sa_email = "${local.sa_account_id}@${var.project_id}.iam.gserviceaccount.com" + oauth_client_id = var.provision_service_account ? google_service_account.connector_sa[0].unique_id : "REPLACE_WITH_NUMERIC_CLIENT_ID_AFTER_CREATING_SERVICE_ACCOUNT" + service_account_email_for_todo = var.provision_service_account ? google_service_account.connector_sa[0].email : local.expected_sa_email } # service account to personify connector +moved { + from = google_service_account.connector_sa + to = google_service_account.connector_sa[0] +} + resource "google_service_account" "connector_sa" { + count = var.provision_service_account ? 1 : 0 + project = var.project_id account_id = local.sa_account_id display_name = var.display_name @@ -31,7 +41,7 @@ resource "google_service_account" "connector_sa" { } resource "google_project_service" "apis_needed" { - for_each = toset(var.apis_consumed) + for_each = var.enable_apis ? toset(var.apis_consumed) : toset([]) service = each.key project = var.project_id @@ -88,33 +98,40 @@ connection to will fail) Account to Use for Connection' setting when they create the connection. 8. Optionally, you may also set the email address of the account you created the value of - `google_workspace_example_user` in your `terraform.tfvars` file. This will cause the example + `google_workspace_connector_settings` (eg, `example_user`) in your `terraform.tfvars` file. This will cause the example API invocations generated by the terraform modules to prefill this value as the account to impersonate on those requests. This will allow you to validate the permissions of the account, as well as the ability of the proxy connection to impersonate it. EOT + manual_sa_todo_note = var.provision_service_account ? "" : <<-EOT + + NOTE: Terraform did not provision this service account. After you create it manually, replace + the placeholder client ID above with the numeric ID shown in the GCP console for + `${local.expected_sa_email}`. +EOT + todo_content = < "Access and Data Control" --> "API Controls", then find "Manage Domain Wide Delegation". Click "Add new". - 2. Copy and paste client ID `${google_service_account.connector_sa.unique_id}` into the + 2. Copy and paste client ID `${local.oauth_client_id}` into the "Client ID" input in the popup. (this is the unique ID of the GCP service account with - email `${google_service_account.connector_sa.email}`; you can (and should) verify its identity - via the GCP console, with the project `${google_service_account.connector_sa.project}`, under: + email `${local.service_account_email_for_todo}`; you can (and should) verify its identity + via the GCP console, with the project `${var.project_id}`, under: - ["IAM & Admin" --> "Service Accounts"](https://console.cloud.google.com/iam-admin/serviceaccounts?project=${google_service_account.connector_sa.project}&supportedpurview=project) + ["IAM & Admin" --> "Service Accounts"](https://console.cloud.google.com/iam-admin/serviceaccounts?project=${var.project_id}&supportedpurview=project) This ensures you are granting domain-wide delegation to the correct service account, and mitigates the risk that these instructions were forged by a malicious actor. - +${local.manual_sa_todo_note} Via the GCP console, you can also verify all extant keys for the service account, to ensure that there is exactly one, which should be held by the proxy. GCP provides log of key usage, creation, revocation, etc, which you can monitor to ensure that the key is being used only by the proxy, only for the data access you expect. If you ever suspect compromise, you may revoke - the key from the GCP console at any time (NOTE: that proxy connection will be broken until your - Terraform configuration is re-applied, to provision a new key). + the key from the GCP console at any time (NOTE: that proxy connection will be broken until a new + key is provisioned and stored in your secrets manager). 3. Copy and paste the following OAuth 2.0 scope string into the "Scopes" input: ``` @@ -122,7 +139,7 @@ ${join(",", var.oauth_scopes_needed)} ``` 4. Authorize it. With this, your psoxy instance should be able to authenticate with Google as - the GCP Service Account `${google_service_account.connector_sa.email}` and request data from + the GCP Service Account `${local.service_account_email_for_todo}` and request data from Google as authorized by the OAuth scopes you granted. ${local.google_workspace_admin_account_required ? local.google_workspace_service_account_setup : ""} EOT diff --git a/infra/modules/google-workspace-dwd-connection/output.tf b/infra/modules/google-workspace-dwd-connection/output.tf index b362346589..2eaec8c4cd 100644 --- a/infra/modules/google-workspace-dwd-connection/output.tf +++ b/infra/modules/google-workspace-dwd-connection/output.tf @@ -3,15 +3,16 @@ output "instance_id" { } output "service_account_id" { - value = google_service_account.connector_sa.id + value = var.provision_service_account ? google_service_account.connector_sa[0].id : "projects/${var.project_id}/serviceAccounts/${local.expected_sa_email}" } output "service_account_email" { - value = google_service_account.connector_sa.email + value = var.provision_service_account ? google_service_account.connector_sa[0].email : local.expected_sa_email } output "service_account_numeric_id" { - value = google_service_account.connector_sa.unique_id + value = var.provision_service_account ? google_service_account.connector_sa[0].unique_id : null + description = "OAuth client ID for domain-wide delegation; null if the service account is not provisioned by Terraform" } output "next_todo_step" { diff --git a/infra/modules/google-workspace-dwd-connection/variables.tf b/infra/modules/google-workspace-dwd-connection/variables.tf index f527babc03..73acc2addf 100644 --- a/infra/modules/google-workspace-dwd-connection/variables.tf +++ b/infra/modules/google-workspace-dwd-connection/variables.tf @@ -36,6 +36,18 @@ variable "oauth_scopes_needed" { default = [] } +variable "provision_service_account" { + type = bool + description = "whether to provision the GCP service account (OAuth client) via Terraform. If false, you must create it manually." + default = true +} + +variable "enable_apis" { + type = bool + description = "whether to enable required GCP APIs via Terraform. If false, you must enable them manually." + default = true +} + variable "todos_as_local_files" { type = bool description = "whether to render TODOs as flat files" @@ -47,4 +59,3 @@ variable "todo_step" { description = "of all todos, where does this one logically fall in sequence" default = 1 } - diff --git a/infra/modules/worklytics-connector-specs/google-workspace.tf b/infra/modules/worklytics-connector-specs/google-workspace.tf index ff13d640d3..fe1daeebfa 100644 --- a/infra/modules/worklytics-connector-specs/google-workspace.tf +++ b/infra/modules/worklytics-connector-specs/google-workspace.tf @@ -1,11 +1,11 @@ locals { google_workspace_example_user = try( - coalesce(var.google_workspace_connector_settings["google_workspace_example_user"]), + coalesce(var.google_workspace_connector_settings["example_user"]), coalesce(var.google_workspace_example_user, "REPLACE_WITH_EXAMPLE_USER@YOUR_COMPANY.COM") ) google_workspace_example_admin = try( - coalesce(var.google_workspace_connector_settings["google_workspace_example_admin"]), + coalesce(var.google_workspace_connector_settings["example_admin"]), coalesce(var.google_workspace_example_admin, local.google_workspace_example_user, "REPLACE_WITH_EXAMPLE_ADMIN@YOUR_COMPANY.COM") ) google_workspace_sources = { diff --git a/infra/modules/worklytics-connector-specs/variables.tf b/infra/modules/worklytics-connector-specs/variables.tf index 10d56bf490..b662de03ec 100644 --- a/infra/modules/worklytics-connector-specs/variables.tf +++ b/infra/modules/worklytics-connector-specs/variables.tf @@ -237,6 +237,6 @@ variable "msft_365_connector_settings" { variable "google_workspace_connector_settings" { type = map(any) - description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." + description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." default = {} } diff --git a/infra/modules/worklytics-connectors-google-workspace/gcp-api-enable-todo.tftpl b/infra/modules/worklytics-connectors-google-workspace/gcp-api-enable-todo.tftpl new file mode 100644 index 0000000000..76bf2cb238 --- /dev/null +++ b/infra/modules/worklytics-connectors-google-workspace/gcp-api-enable-todo.tftpl @@ -0,0 +1,16 @@ +In the GCP console for `${gcp_project_id}` (or via `gcloud`), enable the following APIs required by the `${connector_id}` connector: + +%{ for api in apis_consumed ~} +- `${api}` +%{ endfor ~} + +Via the GCP console: navigate to "APIs & Services" --> "Library", search for each API above, and click "Enable". + +Via gcloud (one command per API): +%{ for api in apis_consumed ~} +`gcloud services enable ${api} --project=${gcp_project_id}` +%{ endfor ~} + +See the page below for more information on provisioning Google Workspace connectors without Terraform: + +https://docs.worklytics.co/psoxy/sources/google-workspace#provisioning-api-clients-without-terraform diff --git a/infra/modules/worklytics-connectors-google-workspace/gcp-sa-create-todo.tftpl b/infra/modules/worklytics-connectors-google-workspace/gcp-sa-create-todo.tftpl new file mode 100644 index 0000000000..461e53cfcc --- /dev/null +++ b/infra/modules/worklytics-connectors-google-workspace/gcp-sa-create-todo.tftpl @@ -0,0 +1,18 @@ +In the GCP console for `${gcp_project_id}` (or via `gcloud`), create a service account to use as the OAuth client for the `${connector_id}` connector: + +- **Account ID**: `${service_account_id}` +- **Display name**: `${display_name}` +- **Description**: `${description}` + +The service account email should be `${expected_service_account_email}`. + +Via the GCP console: navigate to "IAM & Admin" --> "Service Accounts" --> "Create Service Account", and use the values above. + +Via gcloud: +`gcloud iam service-accounts create ${service_account_id} --display-name="${display_name}" --description="${description}" --project=${gcp_project_id}` + +After creating the service account, note its numeric **Client ID** (unique ID) from the service account details page. You will need it when completing domain-wide delegation setup. + +See the page below for more information on provisioning Google Workspace connectors without Terraform: + +https://docs.worklytics.co/psoxy/sources/google-workspace#provisioning-api-clients-without-terraform diff --git a/infra/modules/worklytics-connectors-google-workspace/main.tf b/infra/modules/worklytics-connectors-google-workspace/main.tf index 95a41da4f1..3382cb4006 100644 --- a/infra/modules/worklytics-connectors-google-workspace/main.tf +++ b/infra/modules/worklytics-connectors-google-workspace/main.tf @@ -1,6 +1,18 @@ locals { - provision_gcp_sa_keys = try(var.google_workspace_connector_settings["google_workspace_provision_keys"], var.provision_gcp_sa_keys) - gcp_sa_key_rotation_days = try(var.google_workspace_connector_settings["google_workspace_key_rotation_days"], var.gcp_sa_key_rotation_days) + provision_service_accounts = try(var.google_workspace_connector_settings["provision_service_accounts"], true) + enable_apis = try(var.google_workspace_connector_settings["enable_apis"], true) + provision_gcp_sa_keys = ( + local.provision_service_accounts + ? try(var.google_workspace_connector_settings["provision_keys"], var.provision_gcp_sa_keys) + : false + ) + gcp_sa_key_rotation_days = try(var.google_workspace_connector_settings["key_rotation_days"], var.gcp_sa_key_rotation_days) + + manual_steps_before_dwd = (local.enable_apis ? 0 : 1) + (local.provision_service_accounts ? 0 : 1) + dwd_todo_step = var.todo_step + local.manual_steps_before_dwd + api_todo_step = var.todo_step + sa_todo_step = var.todo_step + (local.enable_apis ? 0 : 1) + key_todo_step = local.dwd_todo_step + 1 } terraform { required_version = "~> 1.7" @@ -46,24 +58,65 @@ module "google_workspace_connection" { description = "Google API OAuth Client for ${each.value.display_name}" apis_consumed = each.value.apis_consumed oauth_scopes_needed = each.value.oauth_scopes_needed + provision_service_account = local.provision_service_accounts + enable_apis = local.enable_apis todos_as_local_files = var.todos_as_local_files - todo_step = var.todo_step + todo_step = local.dwd_todo_step } locals { + api_enable_todos = { + for id, connection in module.google_workspace_connection : + id => templatefile("${path.module}/gcp-api-enable-todo.tftpl", { + gcp_project_id : var.gcp_project_id + connector_id : id + apis_consumed : module.worklytics_connector_specs.enabled_google_workspace_connectors[id].apis_consumed + }) + } + + sa_creation_todos = { + for id, connection in module.google_workspace_connection : + id => templatefile("${path.module}/gcp-sa-create-todo.tftpl", { + gcp_project_id : var.gcp_project_id + connector_id : id + service_account_id : "${local.environment_id_prefix}${substr(id, 0, 30 - length(local.environment_id_prefix))}" + display_name : "Psoxy Connector - ${local.environment_id_display_name_qualifier}${module.worklytics_connector_specs.enabled_google_workspace_connectors[id].display_name}" + description : "Google API OAuth Client for ${module.worklytics_connector_specs.enabled_google_workspace_connectors[id].display_name}" + expected_service_account_email : connection.service_account_email + }) + } + key_creation_todos = { for id, connection in module.google_workspace_connection : id => templatefile("${path.module}/gcp-sa-key-create-todo.tftpl", { gcp_project_id : var.gcp_project_id, gcp_service_account : connection.service_account_email, secret_prefix : connection.instance_id }) } - todos = [for id, connection in module.google_workspace_connection : - local.provision_gcp_sa_keys ? connection.todo : "${local.key_creation_todos[id]}\n${connection.todo}" - ] + connector_todos = { + for id, connection in module.google_workspace_connection : + id => join("\n\n", [for part in [ + local.enable_apis ? null : local.api_enable_todos[id], + local.provision_service_accounts ? null : local.sa_creation_todos[id], + connection.todo, + local.provision_gcp_sa_keys ? null : local.key_creation_todos[id], + ] : part if part != null]) + } + + todos = [for id, connection in module.google_workspace_connection : local.connector_todos[id]] - current_todo_step = try(max(values(module.google_workspace_connection)[*].next_todo_step...), var.todo_step) + current_todo_step = try(max(values(module.google_workspace_connection)[*].next_todo_step...), local.dwd_todo_step) next_todo_step = local.provision_gcp_sa_keys ? local.current_todo_step : local.current_todo_step + 1 + connectors_needing_manual_api_enablement = local.enable_apis ? {} : { + for k, v in module.worklytics_connector_specs.enabled_google_workspace_connectors : + k => v + } + + connectors_needing_manual_sa_creation = local.provision_service_accounts ? {} : { + for k, v in module.worklytics_connector_specs.enabled_google_workspace_connectors : + k => v + } + service_accounts_tf_managed_keys = local.provision_gcp_sa_keys ? { for k, v in module.worklytics_connector_specs.enabled_google_workspace_connectors : k => module.google_workspace_connection[k].service_account_id @@ -75,10 +128,24 @@ locals { } } +resource "local_file" "todo_gcp_api_enablement" { + for_each = var.todos_as_local_files ? local.connectors_needing_manual_api_enablement : {} + + filename = "TODO ${local.api_todo_step} - Enable APIs for ${each.key}.md" + content = local.api_enable_todos[each.key] +} + +resource "local_file" "todo_gcp_sa_creation" { + for_each = var.todos_as_local_files ? local.connectors_needing_manual_sa_creation : {} + + filename = "TODO ${local.sa_todo_step} - Create Service Account for ${each.key}.md" + content = local.sa_creation_todos[each.key] +} + resource "local_file" "todo_gcp_sa_key_creation" { for_each = var.todos_as_local_files ? local.service_accounts_user_managed_keys : {} - filename = "TODO ${local.current_todo_step} - Create Key for ${each.key}.md" + filename = "TODO ${local.key_todo_step} - Create Key for ${each.key}.md" content = local.key_creation_todos[each.key] } diff --git a/infra/modules/worklytics-connectors-google-workspace/variables.tf b/infra/modules/worklytics-connectors-google-workspace/variables.tf index dd039be8c4..2d46c69e39 100644 --- a/infra/modules/worklytics-connectors-google-workspace/variables.tf +++ b/infra/modules/worklytics-connectors-google-workspace/variables.tf @@ -50,7 +50,7 @@ variable "google_workspace_example_admin" { variable "provision_gcp_sa_keys" { type = bool - description = "whether to provision key for each connector's GCP Service Account (OAuth Client). If false, you must create the key manually and provide it." + description = "[DEPRECATED - use google_workspace_connector_settings map instead] whether to provision key for each connector's GCP Service Account (OAuth Client). If false, you must create the key manually and provide it. Ignored if service accounts are not provisioned by Terraform." default = true } @@ -80,6 +80,6 @@ variable "todo_step" { variable "google_workspace_connector_settings" { type = map(any) - description = "Map of configuration settings specifically for Google Workspace connectors (e.g. example users). Note that provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." + description = "Map of configuration settings specifically for Google Workspace connectors. Supported keys: example_user, example_admin, provision_keys, key_rotation_days, provision_service_accounts, enable_apis. Provider-controlling parameters (like GCP project IDs or impersonation SAs) remain top-level variables." default = {} }