Upgrade to OpenSearch (#10)

* upgrade-to-opensearch

LogsToElasticsearch/index.js → CWLogsToOpenSearch/index.js

@@ -8,7 +8,7 @@ var crypto = require('crypto');
 var endpoint = process.env.DOMAIN_ENDPOINT;
 
 // Set this to true if you want to debug why data isn't making it to
-// your Elasticsearch cluster. This will enable logging of failed items
+// your Opensearch cluster. This will enable logging of failed items
 // to CloudWatch Logs.
 var logFailedResponses = false;
 
@@ -23,18 +23,18 @@ exports.handler = function(input, context) {
         // parse the input from JSON
         var awslogsData = JSON.parse(buffer.toString('utf8'));
 
-        // transform the input to Elasticsearch documents
-        var elasticsearchBulkData = transform(awslogsData);
+        // transform the input to Opensearch documents
+        var opensearchBulkData = transform(awslogsData);
 
         // skip control messages
-        if (!elasticsearchBulkData) {
+        if (!opensearchBulkData) {
             console.log('Received a control message');
             context.succeed('Control message handled successfully');
             return;
         }
 
-        // post documents to the Amazon Elasticsearch Service
-        post(elasticsearchBulkData, function(error, success, statusCode, failedItems) {
+        // post documents to the Amazon Opensearch Service
+        post(opensearchBulkData, function(error, success, statusCode, failedItems) {
             console.log('Response: ' + JSON.stringify({
                 "statusCode": statusCode
             }));

monitoring-py/handler.py → CWMetricsToOpenSearch/handler.py

@@ -47,12 +47,11 @@
 DOMAIN_ADMIN_PW = os.environ['DOMAIN_ADMIN_PW']
 REGIONS = json.loads(os.environ['REGIONS'])
 
-
 ################################################################################
 # Timestamp tracking
 
 def get_last_timestamp_ddb(domain_name, region):
-    ddb = boto3.client('dynamodb', 'us-west-2')
+    ddb = boto3.client('dynamodb')
     try:
         ret = ddb.get_item(TableName=DDB_TABLE,
                             Key={'domain': {'S': domain_name},
@@ -65,13 +64,13 @@ def get_last_timestamp_ddb(domain_name, region):
         iso_ts = iso_ts['S']
         return parser.parse(iso_ts)
     except Exception as e:
-        print('Exception retrieving timestap for "{}:{}"'.format(domain_name, region))
+        print('Exception retrieving timestamp for "{}:{}"'.format(domain_name, region))
         print(e)
     return None
 
 
 def update_metric_timestamp_ddb(domain_name, region, ts):
-    ddb = boto3.client('dynamodb', 'us-west-2')
+    ddb = boto3.client('dynamodb')
     try:
         existing = get_last_timestamp_ddb(domain_name, region)
         if not existing or (existing and existing < ts):
@@ -82,7 +81,7 @@ def update_metric_timestamp_ddb(domain_name, region, ts):
                 AttributeUpdates={'Timestamp': { 'Value': {'S': ts.isoformat()}}}
             )
     except Exception as e:
-        print('Exception putting timestap for "{}:{}"'.format(domain_name, region))
+        print('Exception putting timestamp for "{}:{}"'.format(domain_name, region))
         print(e)
 
 
@@ -104,7 +103,6 @@ def get_last_timestamp(domain_name, region):
 
 ################################################################################
 # Domain tracking; 
-# REGIONS = ['us-east-1'] #, 'us-west-2'] # , 'us-east-2', 'us-west-1', 'us-west-2']
 
 def list_all_domains():
     ''' Loops through the list of REGIONS, listing out all domains for this
@@ -255,14 +253,13 @@ def get_all_domain_metric_values(domains):
 
 
 ################################################################################
-# Amazon ES interface
+# Amazon OpenSearch interface
 
 INDEX_DESCRIPTOR = IndexDescriptor(es_index='domains', es_v7=True, timestamped=True)
 ES_AUTH = es_sink.es_auth.ESHttpAuth(DOMAIN_ADMIN_UNAME, DOMAIN_ADMIN_PW)
 ES_DESCRIPTOR = ESDescriptor(
     endpoint=DOMAIN_ENDPOINT,
     index_descriptor=INDEX_DESCRIPTOR,
-    region='us-west-2', 
     auth=ES_AUTH
 )
 ES_BUFFER = es_sink.flushing_buffer.flushing_buffer_factory(ES_DESCRIPTOR,

README.md

@@ -1,62 +1,62 @@
-# Amazon Elasticsearch Service Monitor
+# Amazon OpenSearch Service Monitor
 
-This repository contains step by step demonstration to setup monitoring Stack for Amazon ES domains across all specified regions. This example uses AWS CDK and Python.
+This repository contains step by step demonstration to setup monitoring Stack for Amazon OpenSearch Service domains across all specified regions. This example uses AWS CDK and Python.
 
 
 ## Table of Contents
 1. [Context](#context)
 2. [Prerequisites](#prerequisites)
 3. [Deploy](#deploy)
-4. [Elasticsearch Subscription Filters](#cw-subscription-filters)
+4. [OpenSearch Subscription Filters](#cw-subscription-filters)
 4. [Pre-built Monitoring Dashboards](#dashboards)
 5. [Pre-built Alerts](#alerts)
 6. [Clean up](#cleanup)
 7. [Total Cost of Ownership](#tco)
 
 ## Context <a name="context"></a>
-Amazon Elasticsearch Service is a fully managed service that makes it easy for you to deploy, secure, and run Elasticsearch cost effectively at scale. Customers often have an issue to manage and monitor multiple Amazon ES domains as those metrics and logs are not available at centralized place for troubleshooting the issue. 
-This example helps you to configure a monitoring Amazon ES domains, which will fetch the Cloudwatch Metrics and Cloudwatch logs from all domains at a regular interval. This example also comes with pre-built Kibana dashboards and Alerts. 
+Amazon OpenSearch Service is a fully managed service that makes it easy for you to deploy, secure, and run OpenSearch cost effectively at scale. Customers often have an issue to manage and monitor multiple Amazon OpenSearch Service domains as those metrics and logs are not available at centralized place for troubleshooting the issue. 
+This example helps you to configure a monitoring for Amazon OpenSearch Service domains, which will fetch the Cloudwatch Metrics and Cloudwatch logs from all domains at a regular interval. This example also comes with pre-built OpenSearch dashboards and Alerts. 
 
 ## Architecture
-![architecture](/images/Amazon_ES_Monitoring_Framework.png)
+![architecture](/images/amazon_opensearch_service_monitor_framework.png)
 
 -----
 
 ## Prerequisites <a name="prerequisites"></a>
 
-The following tools are required to deploy this CDK Monitoring tool for Amazon ES.
+The following tools are required to deploy this Monitoring tool for Amazon OpenSearch Service.
 
 AWS CDK - https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html  
 AWS CLI - https://aws.amazon.com/cli/  
 Git -  https://git-scm.com/downloads  
 nodejs - https://nodejs.org/en
 python (3.6 or later) - https://www.python.org/downloads/  
 
-### Create and deploy CDK Monitoring tool
+### Create and deploy Amazon OpenSearch Service Monitor tool
 
-Complete the following steps to set up the CDK Monitoring tool in your environment.
+Complete the following steps to set up the Amazon OpenSearch Service Monitor tool in your environment using CDK.
 
 At a bash terminal session.
 
 ```bash
 # clone the repo
-$ git clone https://github.com/aws-samples/amazon-elasticsearch-service-monitor.git
+$ git clone https://github.com/aws-samples/amazon-opensearch-service-monitor.git
 # move to directory
-$ cd amazon-elasticsearch-service-monitor
+$ cd amazon-opensearch-service-monitor
 ```
 
-![Clone Repo](/images/cdk_monitoring_clone.png)
+![Clone Repo](/images/opensearch_monitor_clone.png)
 
 ```bash
 # bootstrap the remaining setup (assumes us-west-2)
 # Enter the e-mail address for alert, as that will be used for sending the alert
-# Alternatively you can change e-mail address manually in monitoring_cdk/monitoring_cdk_stack.py
+# Alternatively you can change e-mail address manually in opensearch/opensearch_monitor_stack.py
 $ bash bootstrap.sh
 # activate the virtual environment
 $ source .env/bin/activate
 ```
 
-![Bootstrap](/images/cdk_monitoring_bootstrap.png)
+![Bootstrap](/images/opensearch_monitor_bootstrap.png)
 
 ### Bootstrap the CDK
 
@@ -67,25 +67,25 @@ Create the CDK configuration by bootstrapping the CDK.
 (.env)$ cdk bootstrap aws://yourAccountID/yourRegion
 ```
 
-![Terminal - Bootstrap the CDK](/images/cdk_monitoring_bootstrap_cdk.png)
+![Terminal - Bootstrap the CDK](/images/opensearch_monitor_bootstrap_cdk.png)
 
 -----
 
 ## Deploy <a name="deploy"></a>
-Use the AWS CDK to deploy monitoring-cdk stack for Amazon ES. This stack comprises of creating/deploying below components:
+Use the AWS CDK to deploy opensearch stack for Amazon OpenSearch Service. This stack comprises of creating/deploying below components:
 1. Create VPC with 3 AZ
-2. Create and launch Amazon ES cluster (version 7.10) having two t3.medium data nodes with 100GB of EBS storage volume. These 2 nodes are spread across 2 different AZ's
+2. Create and launch Amazon OpenSearch Service cluster (version 1.0) having two t3.medium data nodes with 100GB of EBS storage volume. These 2 nodes are spread across 2 different AZ's
 3. Create Dynamo DB table for timestamp tracking 
 4. Create lambda function to fetch Cloudwatch metrics across all regions and all domains. By default it fetches the data every 5 min, which can be changed if needed. 
-5. Create and launch an EC2 instance which acts as SSH tunnel to access kibana, as all of our setup is secured and in VPC
-6. Create default kibana dashboard to visualize metrics across all domains
-7. Create and setup default e-mail alerts to newly launched Amazon ES cluster
+5. Create and launch an EC2 instance which acts as SSH tunnel to access dashboards, as all of our setup is secured and in VPC
+6. Create default OpenSearch dashboards to visualize metrics across all domains
+7. Create and setup default e-mail alerts to newly launched Amazon OpenSearch Service cluster
 8. Create Index template and Index State Management (ISM) policy to delete indices older than 366 days. (can be changed to different retention if needed)
-9. Monitoring stack has an option to enable Ultra Warm (UW) which is disabled by default, Change settings [in this file](monitoring_cdk/monitoring_cdk_stack.py) to enable UW.
+9. Monitoring stack has an option to enable Ultra Warm (UW) which is disabled by default, Change settings [in this file](opensearch/opensearch_monitor_stack.py) to enable UW.
 10. Create lambda function to fetch Cloudwatch metrics and Cloudwatch logs across all regions.
 
 
-#### Note: Complete stack gets setup with pre-defined configuration defined in [monitoring_cdk_stack.py](monitoring_cdk/monitoring_cdk_stack.py), please review the settings such as e-mail, instance type, username, password before proceeding to deploy. You can also enable UW and dedicated master (if needed)
+#### Note: Complete stack gets setup with pre-defined configuration defined in [opensearch_monitor_stack.py](opensearch/opensearch_monitor_stack.py), please review the settings such as e-mail, instance type, username, password before proceeding to deploy. You can also enable UW and dedicated master (if needed)
 
 Run below command 
 ```bash
@@ -94,45 +94,45 @@ Run below command
 
 The CDK will prompt to apply Security Changes, input "y" for Yes.
 
-![Terminal - Deploy CDK](/images/cdk_monitoring_deploy.png)
+![Terminal - Deploy OpenSearch Monitor Tool](/images/opensearch_monitor_deploy.png)
 
-  Once the app is deployed you will get the Kibana URL, user and password to access Kibana. Once logged in you can refer below sections to navigate around dashboards and alerts.
+  Once the app is deployed you will get the Dashboards URL, user and password to access OpenSearch Dashboards. Once logged in you can refer below sections to navigate around dashboards and alerts.
 
 ####  Note: After the stack is deployed you will recieve an e-mail to confirm the subscription, please confirm the same to start getting the alerts.  
 
 -----
 
-## Post-Deployment: Setup Elasticsearch subscription filters for Cloudwatch logs <a name="cw-subscription-filters"></a>
-  Once stack is deployed successfully you need to create subscription filter and assign them to Lambda. Run [postCDK.py](monitoring_cdk/postCDK.py) to create the subscription filter (assuming the CW log groups with prefix as /aws/aes/domains), if there is any change in prefix please make sure to change above file before running the steps as below.
+## Post-Deployment: Setup OpenSearch subscription filters for Cloudwatch logs <a name="cw-subscription-filters"></a>
+  Once stack is deployed successfully you need to create subscription filter and assign them to Lambda. Run [setupCWSubscriptionFilter.py](opensearch/setupCWSubscriptionFilter.py) to create the subscription filter (assuming the CW log groups with prefix as /aws/aes/domains), if there is any change in prefix please make sure to change above file before running the steps as below.
 
 ```bash
-(.env)$ python3 monitoring_cdk/postCDK.py deploy
+(.env)$ python3 opensearch/setupCWSubscriptionFilter.py deploy
 ```
-![Terminal - Post Deploy CDK](/images/cdk_monitoring_post_deploy.png)
+![Terminal - Setup CW Subscription filter Post Deploy](/images/opensearch_monitor_post_deploy.png)
 -----
 
 ## Pre-built Monitoring Dashboards <a name="dashboards"></a>
-  Monitoring CDK comes with pre-built dashboards which can be accessed as below:
-  1. Login to Kibana: Access kibana with an IP obtained after the deployment and login as below
-      ![Kibana login screen](/images/kibana_login.png)
+  Monitoring domain comes with pre-built dashboards which can be accessed as below:
+  1. Login to Dashboards: Access OpenSearch Dashboards with an IP obtained after the deployment and login as below
+      ![Dashboards login screen](/images/opensearch_dashboards_login.png)
 
-  2. Once logged in, select dashboard as shown below
-      ![Kibana dashboard](/images/kibana_select_dashboard.png)
+  2. Once logged in, select the private tenant from the pop up and then select dashboard as shown below
+      ![OpenSearch Dashboards](/images/opensearch_dashboards_select.png)
 
   3. After clicking on dashboard, it displays list of the dashboard which comes as default
-      ![Kibana dashboard List](/images/kibana_dashboards_list.png)
+      ![OpenSearch Dashboards List](/images/opensearch_dashboards_list.png)
 
-   - **Domain Metrics At A glance** : This gives a 360 degree view of all Amazon ES domains across the regions. 
-      ![Domain Metrics At A glance](/images/dashboard_domain_metrics_at_a_glance.png)
+   - **Domain Metrics At A glance** : This gives a 360 degree view of all Amazon OpenSearch Service domains across the regions. 
+      ![Domain Metrics At A glance](/images/opensearch_domain_metrics_at_a_glance.png)
 
    - **Domain Overview** :  This gives a more detailed metrics for a particular domain, could help to deep dive for issues into a specific domain. 
-      ![Domain Overview](/images/dashboard_domain_overview.png)
+      ![Domain Overview](/images/opensearch_domain_overview.png)
 
 -----
 
 ## Pre-built Alerts <a name="alerts"></a>
 
-  Monioring CDK comes with pre-built alerts as below, which could help to get notified as an email alert for event such as Cluster Health, Disk Issue, Memory Issue , JVM issue etc. 
+  Monioring domains comes with pre-built alerts as below, which could help to get notified as an email alert for event such as Cluster Health, Disk Issue, Memory Issue , JVM issue etc. 
 
 | Alert Type                    | Frequency     |
 | ----------------------------- | ------------- |
@@ -142,26 +142,27 @@ The CDK will prompt to apply Security Changes, input "y" for Yes.
 | JVM Memory Pressure > 80%     | 5 Min         |
 | CPU Utilization > 80%         | 15 Min        |
 | No Kibana Healthy Nodes       | 15 Min        |
+| No Dashboards Healthy Nodes   | 15 Min        |
 | Invalid Host Header Requests  | 15 Min        |
 | Cluster Health - Yellow       | 30 Min        |
 
 -----
 ## Cleanup <a name=cleanup></a>
 
-To clean up the stacks. destroy the monitoring-cdk stack, all other stacks will be torn down due to dependencies. 
+To clean up the stacks. destroy the opensearch stack, all other stacks will be torn down due to dependencies. 
 
 ```bash
 (.env)$ cdk destroy
 ```
 
-![Destroy](/images/cdk_monitoring_destroy.png)
+![Destroy](/images/opensearch_monitor_destroy.png)
 
-To remove subscription for Cloudwatch logs run the script as below. This will traverse the Amazon ES cloudwatch logs and delete any filter which has been created during the deploy.
+To remove subscription for Cloudwatch logs run the script as below. This will traverse the Amazon OpenSearch Service cloudwatch logs and delete any filter which has been created during the deploy.
 
 ```bash
-(.env)$ python3 monitoring_cdk/postCDK.py destroy
+(.env)$ python3 opensearch/setupCWSubscriptionFilter.py destroy
 ```
-![Terminal - Post Destroy CDK](/images/cdk_monitoring_post_destroy.png)
+![Terminal - Post Destroy](/images/opensearch_monitor_post_destroy.png)
 -----
 ## Total Cost of Ownership <a name=tco></a>
 

app.py

@@ -7,10 +7,10 @@
 
 from aws_cdk import core
 
-from monitoring_cdk.monitoring_cdk_stack import MonitoringCdkStack
+from opensearch.opensearch_monitor_stack import OpenSearchMonitor
 
 
 app = core.App()
-MonitoringCdkStack(app, "monitoring-cdk")
+OpenSearchMonitor(app, "opensearch-monitor-stack")
 
 app.synth()

bootstrap.sh

@@ -5,30 +5,30 @@ python3 -m venv .env
 # Install into the virtual environment
 source .env/bin/activate
 # download requirements
-.env/bin/python -m pip install -r requirements.txt
+.env/bin/python -m pip install -r requirements.txt --use-deprecated=legacy-resolver
 # Load dependency for lambda functions
-.env/bin/python -m pip install --target monitoring-py/ -r monitoring-py/requirements.txt
+.env/bin/python -m pip install --target CWMetricsToOpenSearch/ -r CWMetricsToOpenSearch/requirements.txt
 
 # create the key pair
 region_default="us-west-2"
 echo -e
 read -p "Please enter your region to bootstrap the env [$region_default]: " region
 region="${region:-$region_default}"
 
-aws ec2 create-key-pair --key-name aes_cdk_monitoring --query 'KeyMaterial' --output text > aes_cdk_monitoring.pem --region $region
+aws ec2 create-key-pair --key-name amazon_opensearch_monitoring --query 'KeyMaterial' --output text > amazon_opensearch_monitoring.pem --region $region
 # update key_pair permissions
-chmod 400 aes_cdk_monitoring.pem
+chmod 400 amazon_opensearch_monitoring.pem
 # move key_pair to .ssh
-mv -f aes_cdk_monitoring.pem $HOME/.ssh/aes_cdk_monitoring.pem
+mv -f amazon_opensearch_monitoring.pem $HOME/.ssh/amazon_opensearch_monitoring.pem
 # start the ssh agent
 eval `ssh-agent -s`
 # add your key to keychain
-ssh-add -k ~/.ssh/aes_cdk_monitoring.pem 
+ssh-add -k ~/.ssh/amazon_opensearch_monitoring.pem 
 
 # Add e-mail for the notification
 email_default="[email protected]"
 echo -e
 read -p "Please enter an e-mail for alert [$email_default]: " email
 email="${email:-$email_default}"
-sed -i -e 's/[email protected]/'$email'/g' monitoring_cdk/monitoring_cdk_stack.py
+sed -i -e 's/[email protected]/'$email'/g' opensearch/opensearch_monitor_stack.py