diff --git a/app.py b/app.py index b6b6c4c3..bfaa647e 100644 --- a/app.py +++ b/app.py @@ -5,12 +5,12 @@ #!/usr/bin/env python3 -from aws_cdk import core +from aws_cdk import App from opensearch.opensearch_monitor_stack import OpenSearchMonitor -app = core.App() +app = App() OpenSearchMonitor(app, "opensearch-monitor-stack") app.synth() diff --git a/bootstrap.sh b/bootstrap.sh index 19005ce8..d9e7c6ad 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -5,16 +5,16 @@ python3 -m venv .env # Install into the virtual environment source .env/bin/activate # download requirements -.env/bin/python -m pip install -r requirements.txt --use-deprecated=legacy-resolver +.env/bin/python -m pip install -r requirements.txt --upgrade pip # Load dependency for lambda functions .env/bin/python -m pip install --target CWMetricsToOpenSearch/ -r CWMetricsToOpenSearch/requirements.txt # create the key pair -region_default="us-west-2" +region_default="us-east-1" echo -e read -p "Please enter your region to bootstrap the env [$region_default]: " region region="${region:-$region_default}" - +aws configure set default.region $region aws ec2 create-key-pair --key-name amazon_opensearch_monitoring --query 'KeyMaterial' --output text > amazon_opensearch_monitoring.pem --region $region # update key_pair permissions chmod 400 amazon_opensearch_monitoring.pem diff --git a/cdk.json b/cdk.json index bfb835a4..6dbf7fc9 100644 --- a/cdk.json +++ b/cdk.json @@ -1,7 +1,6 @@ { "app": "python3 app.py", "context": { - "@aws-cdk/core:enableStackNameDuplicates": "true", "aws-cdk:enableDiffNoFail": "true", "@aws-cdk/core:stackRelativeExports": "true", "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true diff --git a/opensearch/create_alerts.sh b/opensearch/create_alerts.sh index 52014438..d6f8a69a 100644 --- a/opensearch/create_alerts.sh +++ b/opensearch/create_alerts.sh @@ -9,7 +9,7 @@ sleep 10; # There was a bug in elastic which can't store the origin url when string field is converted to URL type and it is exported to another domain. Workaround is to replace it manually. # https://github.com/elastic/kibana/issues/63924 InstanceIP=`curl ifconfig.me` -sed -i 's/CHANGE_ORIGIN_URL/'$InstanceIP'/g' /tmp/assets/export_opensearch_dashboards_V1_0.ndjson +sed -i 's/CHANGE_ORIGIN_URL/'$InstanceIP'/g' /home/ec2-user/assets/export_opensearch_dashboards_V1_0.ndjson # Create backend role to load CW logs using lambda curl -s -XPATCH -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_security/api/rolesmapping/all_access' -H 'Content-Type: application/json' -d '[ {"op":"add","path":"/backend_roles","value":["LAMBDA_CW_LOGS_ROLE_ARN"]} ] ' diff --git a/opensearch/opensearch_monitor_stack.py b/opensearch/opensearch_monitor_stack.py index 294204a0..329dc108 100644 --- a/opensearch/opensearch_monitor_stack.py +++ b/opensearch/opensearch_monitor_stack.py @@ -12,10 +12,11 @@ aws_lambda as lambda_, aws_ec2 as ec2, aws_sns as sns, - aws_sns_subscriptions as subscriptions, - core + aws_sns_subscriptions as subscriptions, + Aws, CfnOutput, Stack, RemovalPolicy, SecretValue, Duration ) from aws_cdk.aws_s3_assets import Asset +from constructs import Construct import boto3 import fileinput import json @@ -65,8 +66,8 @@ # DDB settings TABLE_NAME = 'timestamps' -class OpenSearchMonitor(core.Stack): - def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: +class OpenSearchMonitor(Stack): + def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ################################################################################ @@ -84,9 +85,9 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: es_sec_grp.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(443)) domain = opensearch.Domain(self, 'opensearch-service-monitor', - version=opensearch.EngineVersion.OPENSEARCH_1_0, # Upgrade when CDK upgrades + version=opensearch.EngineVersion.OPENSEARCH_1_2, # Upgrade when CDK upgrades domain_name=DOMAIN_NAME, - removal_policy=core.RemovalPolicy.DESTROY, + removal_policy=RemovalPolicy.DESTROY, capacity=opensearch.CapacityConfig( data_node_instance_type=DOMAIN_DATA_NODE_INSTANCE_TYPE, data_nodes=DOMAIN_DATA_NODE_INSTANCE_COUNT, @@ -115,15 +116,15 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: use_unsigned_basic_auth=True, fine_grained_access_control={ "master_user_name": DOMAIN_ADMIN_UNAME, - "master_user_password": core.SecretValue.plain_text(DOMAIN_ADMIN_PW) + "master_user_password": SecretValue.unsafe_plain_text(DOMAIN_ADMIN_PW) } ) - core.CfnOutput(self, "MasterUser", + CfnOutput(self, "MasterUser", value=DOMAIN_ADMIN_UNAME, description="Master User Name for Amazon OpenSearch Service") - core.CfnOutput(self, "MasterPW", + CfnOutput(self, "MasterPW", value=DOMAIN_ADMIN_PW, description="Master User Password for Amazon OpenSearch Service") @@ -139,7 +140,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: name='region', type=ddb.AttributeType.STRING ), - removal_policy=core.RemovalPolicy.DESTROY + removal_policy=RemovalPolicy.DESTROY ) ################################################################################ @@ -148,10 +149,10 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: self, 'CWMetricsToOpenSearch', function_name="CWMetricsToOpenSearch_monitoring", runtime = lambda_.Runtime.PYTHON_3_8, - code=lambda_.Code.asset('CWMetricsToOpenSearch'), + code=lambda_.Code.from_asset('CWMetricsToOpenSearch'), handler='handler.handler', memory_size=1024, - timeout=core.Duration.minutes(10), + timeout=Duration.minutes(10), vpc=vpc ) @@ -171,7 +172,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: lambda_func.add_to_role_policy(iam.PolicyStatement(actions=['cloudwatch:*'], resources=['*'])) - lambda_schedule = events.Schedule.rate(core.Duration.seconds(LAMBDA_INTERVAL)) + lambda_schedule = events.Schedule.rate(Duration.seconds(LAMBDA_INTERVAL)) event_lambda_target = targets.LambdaFunction(handler=lambda_func) events.Rule( self, @@ -186,7 +187,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: self, 'CWLogsToOpenSearch', function_name="CWLogsToOpenSearch_monitoring", runtime = lambda_.Runtime.NODEJS_12_X, - code=lambda_.Code.asset('CWLogsToOpenSearch'), + code=lambda_.Code.from_asset('CWLogsToOpenSearch'), handler='index.handler', vpc=vpc ) @@ -288,21 +289,22 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: "yum update -y", "yum install jq -y", "amazon-linux-extras install nginx1.12", - "cd /tmp/assets", + "mkdir -p /home/ec2-user/assets", + "cd /home/ec2-user/assets", "mv {} export_opensearch_dashboards_V1_0.ndjson".format(dashboards_asset_path), "mv {} nginx_opensearch.conf".format(nginx_asset_path), "mv {} create_alerts.sh".format(alerting_asset_path), "openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/nginx/cert.key -out /etc/nginx/cert.crt -subj /C=US/ST=./L=./O=./CN=.\n" "cp nginx_opensearch.conf /etc/nginx/conf.d/", - "sed -i 's/DEFAULT_DOMAIN_NAME/" + DOMAIN_NAME + "/g' /tmp/assets/export_opensearch_dashboards_V1_0.ndjson", + "sed -i 's/DEFAULT_DOMAIN_NAME/" + DOMAIN_NAME + "/g' /home/ec2-user/assets/export_opensearch_dashboards_V1_0.ndjson", "sed -i 's/DOMAIN_ENDPOINT/" + domain.domain_endpoint + "/g' /etc/nginx/conf.d/nginx_opensearch.conf", - "sed -i 's/DOMAIN_ENDPOINT/" + domain.domain_endpoint + "/g' /tmp/assets/create_alerts.sh", - "sed -i 's=LAMBDA_CW_LOGS_ROLE_ARN=" + lambda_func_cw_logs.role.role_arn + "=g' /tmp/assets/create_alerts.sh", - "sed -i 's=SNS_ROLE_ARN=" + sns_role.role_arn + "=g' /tmp/assets/create_alerts.sh", - "sed -i 's/SNS_TOPIC_ARN/" + sns_topic.topic_arn + "/g' /tmp/assets/create_alerts.sh", - "sed -i 's=DOMAIN_ADMIN_UNAME=" + DOMAIN_ADMIN_UNAME + "=g' /tmp/assets/create_alerts.sh", - "sed -i 's=DOMAIN_ADMIN_PW=" + DOMAIN_ADMIN_PW + "=g' /tmp/assets/create_alerts.sh", + "sed -i 's/DOMAIN_ENDPOINT/" + domain.domain_endpoint + "/g' /home/ec2-user/assets/create_alerts.sh", + "sed -i 's=LAMBDA_CW_LOGS_ROLE_ARN=" + lambda_func_cw_logs.role.role_arn + "=g' /home/ec2-user/assets/create_alerts.sh", + "sed -i 's=SNS_ROLE_ARN=" + sns_role.role_arn + "=g' /home/ec2-user/assets/create_alerts.sh", + "sed -i 's/SNS_TOPIC_ARN/" + sns_topic.topic_arn + "/g' /home/ec2-user/assets/create_alerts.sh", + "sed -i 's=DOMAIN_ADMIN_UNAME=" + DOMAIN_ADMIN_UNAME + "=g' /home/ec2-user/assets/create_alerts.sh", + "sed -i 's=DOMAIN_ADMIN_PW=" + DOMAIN_ADMIN_PW + "=g' /home/ec2-user/assets/create_alerts.sh", "systemctl restart nginx.service", "chmod 500 create_alerts.sh", @@ -310,11 +312,11 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: "bash --verbose create_alerts.sh", ) - core.CfnOutput(self, "Dashboards URL (via Jump host)", + CfnOutput(self, "Dashboards URL (via Jump host)", value="https://" + instance.instance_public_ip, description="Dashboards URL via Jump host") - core.CfnOutput(self, "SNS Subscription Alert Message", + CfnOutput(self, "SNS Subscription Alert Message", value=SNS_NOTIFICATION_EMAIL, description="Please confirm your SNS subscription receievedt at") diff --git a/requirements.txt b/requirements.txt index 4c88c555..d3393009 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,78 +1,3 @@ -astroid==2.8.0 -attrs==21.2.0 -aws-cdk.assets==1.124.0 -aws-cdk.aws-apigateway==1.124.0 -aws-cdk.aws-apigatewayv2==1.124.0 -aws-cdk.aws-applicationautoscaling==1.124.0 -aws-cdk.aws-autoscaling==1.124.0 -aws-cdk.aws-autoscaling-common==1.124.0 -aws-cdk.aws-autoscaling-hooktargets==1.124.0 -aws-cdk.aws-batch==1.124.0 -aws-cdk.aws-certificatemanager==1.124.0 -aws-cdk.aws-cloudformation==1.124.0 -aws-cdk.aws-cloudfront==1.124.0 -aws-cdk.aws-cloudwatch==1.124.0 -aws-cdk.aws-codebuild==1.124.0 -aws-cdk.aws-codecommit==1.124.0 -aws-cdk.aws-codeguruprofiler==1.124.0 -aws-cdk.aws-codepipeline==1.124.0 -aws-cdk.aws-cognito==1.124.0 -aws-cdk.aws-dynamodb==1.124.0 -aws-cdk.aws-ec2==1.124.0 -aws-cdk.aws-ecr==1.124.0 -aws-cdk.aws-ecr-assets==1.124.0 -aws-cdk.aws-ecs==1.124.0 -aws-cdk.aws-efs==1.124.0 -aws-cdk.aws-elasticloadbalancing==1.124.0 -aws-cdk.aws-elasticloadbalancingv2==1.124.0 -aws-cdk.aws-elasticsearch==1.124.0 -aws-cdk.aws-opensearchservice==1.124.0 -aws-cdk.aws-events==1.124.0 -aws-cdk.aws-events-targets==1.124.0 -aws-cdk.aws-iam==1.124.0 -aws-cdk.aws-kinesis==1.124.0 -aws-cdk.aws-kinesisfirehose==1.124.0 -aws-cdk.aws-kms==1.124.0 -aws-cdk.aws-lambda==1.124.0 -aws-cdk.aws-logs==1.124.0 -aws-cdk.aws-route53==1.124.0 -aws-cdk.aws-route53-targets==1.124.0 -aws-cdk.aws-s3==1.124.0 -aws-cdk.aws-s3-assets==1.124.0 -aws-cdk.aws-sam==1.124.0 -aws-cdk.aws-secretsmanager==1.124.0 -aws-cdk.aws-servicediscovery==1.124.0 -aws-cdk.aws-sns==1.124.0 -aws-cdk.aws-sns-subscriptions==1.124.0 -aws-cdk.aws-sqs==1.124.0 -aws-cdk.aws-ssm==1.124.0 -aws-cdk.aws-stepfunctions==1.124.0 -aws-cdk.cloud-assembly-schema==1.124.0 -aws-cdk.core==1.124.0 -aws-cdk.custom-resources==1.124.0 -aws-cdk.cx-api==1.124.0 -aws-cdk.region-info==1.124.0 -aws-lambda==2.1.3 -boto3==1.16.25 -botocore==1.19.25 -cattrs==1.8.0 -cfn-flip==1.2.3 -cfnresponse==1.1.1 -click==7.1.2 -constructs==3.3.69 -isort==5.6.4 -jmespath==0.10.0 -jsii==1.34.0 -lazy-object-proxy==1.4.3 -mccabe==0.6.1 -publication==0.0.3 -pylint==2.6.0 -python-dateutil==2.8.1 -PyYAML==5.4 -s3transfer==0.3.3 -six==1.15.0 -toml==0.10.2 -troposphere==2.6.3 -typing-extensions==3.7.4.3 -urllib3==1.26.5 -wrapt==1.12.1 +aws-cdk-lib>=2.12.0 +constructs>=10.0.0 +boto3 \ No newline at end of file