Merge pull request #20 from jason-chong/update4-28-23

gh-workload expansion merge 4-28-23
oci-landing-zones · Apr 28, 2023 · 7b9aa30 · 7b9aa30
2 parents 7f7bad1 + a0831a0
commit 7b9aa30
Show file tree

Hide file tree

Showing 37 changed files with 636 additions and 299 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,6 +24,8 @@ terraform.tfstate
 terraform.tfvars
 .terraform.tfstate.lock.info
 
+test/terraform/*/provider.tf
+
 .tfcache
 **/.terraform.lock.hcl
 **/tf_resources.json

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -7,6 +7,8 @@ stages:          # List of stages for jobs, and their order of execution
   - test-unit
   - TF-Plan-e2e
   - TF-Apply-e2e
+  - TF-Workload-Apply-e2e
+  - TF-Reapply-e2e
   - TF-Destroy-e2e
 
 
@@ -30,7 +32,7 @@ unit-test-job:   # This job runs in the test stage.
   extends: .testing_pipeline
   stage: test-unit
   script:
-    - pytest  -m unit --junit-xml=pytest.xml test
+    - pytest  -c test/pytest.ini -m "unit and not expensive" --junit-xml=pytest.xml
   # # Orahub's artifact upload seems broken. Possibly a config issue...  
   # artifacts:
   #   when: always
@@ -39,7 +41,7 @@ unit-test-job:   # This job runs in the test stage.
   #   reports:
   #     junit: pytest.xml
   rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: ($CI_COMMIT_BRANCH != "main")
 
 e2e-full-plan-job:   # This job runs terraform plan.
   extends: .testing_pipeline
@@ -49,9 +51,13 @@ e2e-full-plan-job:   # This job runs terraform plan.
     - cd templates/enterprise-landing-zone
     - ./BackEnd.sh
     - cat backend.tf
+    - curl $BASE_LAST_OUTPUT > base_output.json 
+    - curl $WE_LAST_OUTPUT > we_output.json 
+    - cp $ENV_TFVARS_FILE_NAME  updated-$ENV_TFVARS_FILE_NAME
+    - python3 read_workload_vars.py -f base_output.json -p we_output.json >> updated-$ENV_TFVARS_FILE_NAME
     - terraform init -backend-config="address=$REMOTE_STATE_BACKEND" -backend-config="update_method=PUT"
     - terraform validate
-    - terraform plan -var-file=$ENV_TFVARS_FILE_NAME
+    - terraform plan -var-file=updated-$ENV_TFVARS_FILE_NAME
   rules:
     - if: ($CI_COMMIT_BRANCH == "LANZ-Integration")
 
@@ -63,13 +69,23 @@ e2e-full-deploy-job:   # This job runs terraform apply.
     - cd templates/enterprise-landing-zone
     - ./BackEnd.sh
     - cat backend.tf
+    - curl $BASE_LAST_OUTPUT > base_output.json 
+    - curl $WE_LAST_OUTPUT > we_output.json 
+    - cp $ENV_TFVARS_FILE_NAME  updated-$ENV_TFVARS_FILE_NAME
+    - python3 read_workload_vars.py -f base_output.json -p we_output.json >> updated-$ENV_TFVARS_FILE_NAME
     - terraform init -backend-config="address=$REMOTE_STATE_BACKEND" -backend-config="update_method=PUT"
     - terraform validate
-    - terraform apply -var-file=$ENV_TFVARS_FILE_NAME -auto-approve #neeed to look at pasing a plan file but have to understan gitlab artifiacts issue
+    - terraform apply -var-file=updated-$ENV_TFVARS_FILE_NAME -auto-approve #neeed to look at pasing a plan file but have to understan gitlab artifiacts issue
+    - terraform output -json > base_output.json
+    - 'curl -X PUT -H "Content-Type: application/json"  -d @base_output.json $BASE_LAST_OUTPUT'
   rules:
     - if: ($CI_COMMIT_BRANCH == "LANZ-Integration")
   retry: 1
 
+e2e-redeploy-job:
+  extends: e2e-full-deploy-job
+  stage: TF-Reapply-e2e
+
 e2e-full-destroy-job:   # This job runs terraform destroy.
   extends: .testing_pipeline
   stage: TF-Destroy-e2e
@@ -79,8 +95,36 @@ e2e-full-destroy-job:   # This job runs terraform destroy.
     - cd templates/enterprise-landing-zone
     - ./BackEnd.sh
     - cat backend.tf
+    - curl $BASE_LAST_OUTPUT > base_output.json 
+    - curl $WE_LAST_OUTPUT > we_output.json 
+    - cp $ENV_TFVARS_FILE_NAME  updated-$ENV_TFVARS_FILE_NAME
+    - python3 read_workload_vars.py -f base_output.json -p we_output.json >> updated-$ENV_TFVARS_FILE_NAME
     - terraform init -backend-config="address=$REMOTE_STATE_BACKEND" -backend-config="update_method=PUT"
     - terraform validate
-    - terraform destroy -var-file=$ENV_TFVARS_FILE_NAME -auto-approve #neeed to look at pasing a plan file but have to understand gitlab artifiacts issue
+    - terraform destroy -var-file=updated-$ENV_TFVARS_FILE_NAME -auto-approve #neeed to look at pasing a plan file but have to understand gitlab artifiacts issue
+    - terraform output -json > base_output.json
+    - 'curl -X PUT -H "Content-Type: application/json"  -d @base_output.json $BASE_LAST_OUTPUT'
   rules:
-    - if: ($CI_COMMIT_BRANCH == "LANZ-Integration")
+    - if: ($CI_COMMIT_BRANCH == "LANZ-Integration")
+
+e2e-workload-deploy-job:   # This job runs terraform apply on Workload Expansion.
+  extends: .testing_pipeline
+  stage: TF-Workload-Apply-e2e
+  when: on_success
+  script:
+    - cd templates
+    - elz-workload/add_workload.sh TestWE P
+    - cd elz-workload-P-TestWE
+    - ./BackEnd.sh
+    - cat backend.tf
+    - curl $BASE_LAST_OUTPUT > base_output.json 
+    - curl $WE_LAST_OUTPUT > we_output.json 
+    - python3 read_base_vars.py -e P -f base_output.json >> $ENV_TFVARS_FILE_NAME
+    - terraform init -backend-config="address=$WE_REMOTE_STATE_BACKEND" -backend-config="update_method=PUT"
+    - terraform validate
+    - terraform apply -var-file=$ENV_TFVARS_FILE_NAME -auto-approve #need to look at pasing a plan file but have to understan gitlab artifiacts issue
+    - terraform output -json > we_output.json
+    - 'curl -X PUT -H "Content-Type: application/json"  -d @we_output.json $WE_LAST_OUTPUT'
+  rules:
+    - if: ($CI_COMMIT_BRANCH == "LANZ-Integration")
+  retry: 1
diff --git a/README.md b/README.md
@@ -2,18 +2,25 @@
 
 This repository contains different types of Landing Zones to deploy to the Oracle Cloud Infrastructure platform. The landing zones are assembled from modules and templates that users can use in their default configuration or fork this repo and customize for your own scenarios.
 
-## Enterprise Scale Baseline Landing Zone v2.0 (ESBLZ v2.0)
+## Oracle Enterprise Landing Zone v2 (OELZ v2)
 
-The Enterprise Scale Baseline Landing Zone v2.0 (ESBLZ v2.0) deploys a typical architecture used by enterprise customers. The root template for this landing zone is located at [templates/enterprise-landing-zone](./templates/enterprise-landing-zone). Users can use the guides below to get strated with the Enterprise Scale Baseline Landing Zone v2.0 (ESBLZ v2.0).
+The Oracle Enterprise Landing Zone v2 (OELZ v2) deploys a typical architecture used by enterprise customers. The root template for this landing zone is located at [templates/enterprise-landing-zone](./templates/enterprise-landing-zone). Users can use the guides below to get started with the Oracle Enterprise Landing Zone v2 (OELZ v2).
 
 - [Architecture Guide](./templates/enterprise-landing-zone/Architecture_Guide.md)
 - [Implementation Guide](./templates/enterprise-landing-zone/IMPLEMENTATION.md)
 - [Configuration Guide](./templates/enterprise-landing-zone/CONFIGURATION.md)
 
+### Workload Expansion
+The Oracle Enterprise Landing Zone v2 (OELZ v2) deploys a workload in each environment (Prod and Non-Prod) by default.
+The user can use the workload expansion stack to deploy additional customized workload. The template for the workload expansion is located
+at [templates/elz-workload](./templates/elz-workload). Users can use the guides below to get started with Workload Expansion.
+- [Implementation Guide](./templates/elz-workload/IMPLEMENTATION.md)
+- [Configuration Guide](./templates/elz-workload/CONFIGURATION.md)
+
 ## Deploy Using Oracle Resource Manager
 1. Click to deploy the stack
 
-[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-quickstart/oci-landing-zones/archive/refs/tags/v2.0.0.zip)
+[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-quickstart/oci-landing-zones/archive/refs/tags/v2.zip)
 
     If you aren't already signed in, when prompted, enter the tenancy and user credentials. Review and accept the terms and conditions.
 

diff --git a/RELEASE.md b/RELEASE.md
@@ -3,11 +3,11 @@
 ----
 ## v2.0.0 - 2023-02-28
 - Initial Release of new version 2 codebase with Hub and Spoke Networking, Multi-Environment support and more modular architecture. see the [Architecture Guide](./templates/enterprise-landing-zone/Architecture_Guide.md) for details.
-- CIS Security Benchmark Compliance: Enterprise Scale Baseline Landing Zone v2 was designed to include a foundational set of security controls from the Center for Internet Security (CIS). We are happy to share that this release of Landing Zones will support the recommended CIS 1.2 Level 1 controls. The security controls implemented by this Landing Zone are prescriptive and practical in nature with the primary focus to help provide best practices for security hardening of the technologies that are deployed in our customers' tenancies.
+- CIS Security Benchmark Compliance: Oracle Enterprise Landing Zone v2 was designed to include a foundational set of security controls from the Center for Internet Security (CIS). We are happy to share that this release of Landing Zones will support the recommended CIS 1.2 Level 1 controls. The security controls implemented by this Landing Zone are prescriptive and practical in nature with the primary focus to help provide best practices for security hardening of the technologies that are deployed in our customers' tenancies.
 While many of the CIS Level 1 recommendations are included in the Landing Zone deployment, however, there are some that require administrators to configure manually. Please be advised that for recommendations # 1.5 - 1.13, 2.6 - 2.8 and 3.16, it will be the customer administrators' responsibility to implement and enforce.
 For recommendation #1.7, we recommend that Multi-Factor Authentication (MFA) be fully tested before restricting access only to MFA-verified users. Please note each user must enable MFA for themselves and an administrator cannot enable MFA for another user. For more information, please see [OCI Managing Multi-Factor Authentication documentation][v2.0.0-1].
 For more information on the CIS Security Benchmark, please visit the official [Oracle Cloud CIS Benchmark site][v2.0.0-2].
-- Certain CIDR ranges should not be used when deploying ESBLZv2, as the can conflict with IP addresses reserved for special use. These are:
+- Certain CIDR ranges should not be used when deploying OELZv2, as the can conflict with IP addresses reserved for special use. These are:
     * 169.254.10.0-169.254.19.255
     * 169.254.100.0-169.254.109.255
     * 169.254.192.0-169.254.201.255

diff --git a/templates/README.md b/templates/README.md
@@ -4,4 +4,4 @@ This templates directory holds the base composition modules(templates) for our
 Enterprise Landing Zone. These templates consist of individual definitions of our 
 intended OELZ configuration. As they are individual entries they can be combined or 
 iterated over to allow for creation of multiple infrastructure pieces. The [enterprise-landing-zone](./enterprise-landing-zone/) directory 
-contains the template for Oracle Enterprise Scale Baseline Landing Zone v2.0.
+contains the template for Oracle Enterprise Landing Zone v2.
diff --git a/templates/elz-environment/logging-variables.tf b/templates/elz-environment/logging-variables.tf
@@ -7,3 +7,8 @@ variable "retention_policy_duration_time_unit" {
   type        = string
   description = "The unit that should be used to interpret timeAmount."
 }
+
+variable "is_service_connector_limit" {
+  type        = bool
+  description = "Restrict Number of Service Connector Deployment"
+}
diff --git a/templates/elz-environment/logging.tf b/templates/elz-environment/logging.tf
@@ -5,6 +5,7 @@ module "logging" {
   tenancy_ocid                        = var.tenancy_ocid
   home_compartment_id                 = var.home_compartment_id
   home_compartment_name               = var.home_compartment_name
+  is_service_connector_limit          = var.is_service_connector_limit
   security_compartment_id             = module.compartment.compartments.security.id
   master_encryption_key               = module.security.key_id
   logging_compartment_id              = module.compartment.compartments.logging.id

diff --git a/templates/elz-environment/outputs.tf b/templates/elz-environment/outputs.tf
@@ -31,6 +31,17 @@ output "workload_compartment_id" {
   value = module.workload.compartment_id
 }
 
+output "workload_compartment_name" {
+  value = module.workload.compartment_name
+}
+
+output "workload_subnet_cidr_blocks" {
+  value = [var.private_spoke_subnet_app_cidr_block,
+           var.private_spoke_subnet_db_cidr_block,
+           var.private_spoke_subnet_web_cidr_block,
+  ]
+}
+
 output "stream_id" {
   value = module.logging.stream_id
 }

diff --git a/templates/elz-logging/main.tf b/templates/elz-logging/main.tf
@@ -70,6 +70,9 @@ locals {
     AUDIT: "${var.resource_label}_${var.environment_prefix}_auditLogs_standard",
     SERVICE_EVENT: "${var.resource_label}_${var.environment_prefix}_serviceEvents_standard"
   }
+  buckets_map_service_conector_limit = {
+    SERVICE_EVENT: "${var.resource_label}_${var.environment_prefix}_serviceEvents_standard"
+  }
 
   os_read_log = {
     log_display_name    = "${var.resource_label}-OCI-ELZ-OS-READ-LOG-${var.environment_prefix}"
@@ -139,6 +142,7 @@ module "service_event_stream" {
 }
 
 module "audit_log_bucket" {
+  count                               = var.is_service_connector_limit   ? 0 : 1
   source                              = "../../modules/bucket"
   tenancy_ocid                        = var.tenancy_ocid
   compartment_id                      = var.logging_compartment_id
@@ -152,6 +156,7 @@ module "audit_log_bucket" {
 }
 
 module "default_log_bucket" {
+  count                               = var.is_service_connector_limit   ? 0 : 1
   source                              = "../../modules/bucket"
   tenancy_ocid                        = var.tenancy_ocid
   compartment_id                      = var.logging_compartment_id
@@ -178,6 +183,7 @@ module "service_event_log_bucket" {
 }
 
 module "audit_log_service_connector" {
+  count                 = var.is_service_connector_limit   ? 0 : 1
   source                = "../../modules/service-connector"
   tenancy_ocid          = var.tenancy_ocid
   compartment_id        = var.security_compartment_id
@@ -192,6 +198,7 @@ module "audit_log_service_connector" {
 }
 
 module "default_log_service_connector" {
+  count                 = var.is_service_connector_limit   ? 0 : 1
   source                = "../../modules/service-connector"
   tenancy_ocid          = var.tenancy_ocid
   compartment_id        = var.security_compartment_id
@@ -229,7 +236,8 @@ resource "time_sleep" "first_log_delay" {
 module "os_read_log" {
   source = "../../modules/service-log"
 
-  service_log_map     = local.buckets_map
+  #service_log_map     = local.buckets_map
+  service_log_map     = var.is_service_connector_limit == true ? local.buckets_map_service_conector_limit : local.buckets_map
   log_display_name    = local.os_read_log.log_display_name
   log_type            = local.os_read_log.log_type
   log_group_id        = module.default_log_group.log_group_id
@@ -243,7 +251,8 @@ module "os_read_log" {
 module "os_write_log" {
   source = "../../modules/service-log"
 
-  service_log_map     = local.buckets_map
+  #service_log_map     = local.buckets_map
+  service_log_map     = var.is_service_connector_limit == true ? local.buckets_map_service_conector_limit : local.buckets_map
   log_display_name    = local.os_write_log.log_display_name
   log_type            = local.os_write_log.log_type
   log_group_id        = module.default_log_group.log_group_id

diff --git a/templates/elz-logging/variables.tf b/templates/elz-logging/variables.tf
@@ -57,3 +57,7 @@ variable "subnets_map" {
   type        = map(string)
   description = ""
 }
+variable "is_service_connector_limit" {
+  type        = bool
+  description = "Restrict Number of Service Connector Deployment"
+}
diff --git a/templates/elz-workload/Architecture_Guide.md b/templates/elz-workload/Architecture_Guide.md