feat: Add vpc to all lambdas, allow users to self manage install bucket assets (#15)

* feat: Add vpc to all lambdas, allow users to self manage installastion bucket assets

* docs: Update readme for selfManageInstallationBucketAssets config

Author: sydefz

README.md

@@ -235,6 +235,23 @@ The Federal Information Processing Standard ([FIPS](https://aws.amazon.com/compl
 "useFipsEndpoint": true,
 ```
 
+**Self-managed druid installation bucket asset (optional)**
+
+By default, the solution will upload druid binary, extension, config and scripts to s3 bucket automatically, optionally you can manage these assets yourself by using below configuration
+
+```
+"selfManageInstallationBucketAssets": true,
+```
+When this option is enabled, you will need to manually upload assets in the following folders to s3 bucket after you `npm run build`
+```
+source/lib/uploads/scripts/ -> bootstrap-s3-bucket-installation/scripts/
+source/lib/docker/extensions/ -> bootstrap-s3-bucket-installation/extensions/
+source/lib/uploads/config/ -> bootstrap-s3-bucket-installation/config/
+source/lib/docker/ca-certs/ -> bootstrap-s3-bucket-installation/ca-certs/
+source/druid-bin/ -> bootstrap-s3-bucket-installation/druid-images/
+source/zookeeper-bin/ -> bootstrap-s3-bucket-installation/zookeeper-images/
+```
+
 **Identity provider configuration (optional)**
 The default setup of this solution activates basic authentication, enabling access to the Druid web console and API through a username and password. Additionally, you have the option to enable user access federation through a third-party identity provider (IdP) that supports OpenID Connect authentication. If you have an existing enterprise Identity Provider, you can seamlessly integrate it using the CDK configuration outlined below.
 

source/bin/druid-infra.ts

@@ -84,6 +84,7 @@ const contextKeys: (keyof DruidConfig)[] = [
     'druidInstanceIamPolicyArns',
     'tags',
     'additionalTags',
+    'selfManageInstallationBucketAssets',
 ];
 
 const configMap: Record<string, unknown> = {};
@@ -171,6 +172,7 @@ const commonStackProps = {
     customAmi: druidConfig.customAmi,
     subnetMappings: druidConfig.subnetMappings,
     enableVulnerabilityScanJob: druidConfig.enableVulnerabilityScanJob ?? false,
+    selfManageInstallationBucketAssets: druidConfig.selfManageInstallationBucketAssets ?? false,
     ...(!druidConfig.environmentAgnostic && { env: { account, region } }),
 };
 

source/lib/constructs/appRegistryAspect.ts

@@ -17,6 +17,7 @@ import * as appInsights from 'aws-cdk-lib/aws-applicationinsights';
 import * as appRegistry from '@aws-cdk/aws-servicecatalogappregistry-alpha';
 import * as cdk from 'aws-cdk-lib';
 import * as cr from 'aws-cdk-lib/custom-resources';
+import * as ec2 from 'aws-cdk-lib/aws-ec2';
 import * as iam from 'aws-cdk-lib/aws-iam';
 import * as lambda from 'aws-cdk-lib/aws-lambda';
 import * as path from 'path';
@@ -32,6 +33,7 @@ const DEFAULT_APPLICATION_RESOURCE_GROUP_STATE_QUERY_TIMEOUT = cdk.Duration.minu
 const DEFAULT_APPLICATION_RESOURCE_GROUP_STATE_QUERY_INTERVAL = cdk.Duration.seconds(1);
 
 interface AppRegistryProps {
+    vpc: ec2.IVpc;
     solutionName: string;
     solutionId: string;
     solutionVersion: string;
@@ -52,7 +54,7 @@ export class AppRegistry extends Construct implements cdk.IAspect {
         this.application = this.createAppForAppRegistry();
         this.createAttributeGroup(this.application);
         this.addTagsforApplication(this.application);
-        const waiter = this.waitForResourceGroupCreated(this.application);
+        const waiter = this.waitForResourceGroupCreated(this.application, props.vpc);
 
         this.createAppForAppInsights(this.application, waiter);
     }
@@ -102,7 +104,8 @@ export class AppRegistry extends Construct implements cdk.IAspect {
     // The Application does not expose the resource group instance that we can use to set dependency which will cause the intermittent failure in AppInsight Application provision.
     // Add a waiter customer resource to ensure the Resource Group is CREATED.
     private waitForResourceGroupCreated(
-        application: appRegistry.Application
+        application: appRegistry.Application,
+        vpc: ec2.IVpc,
     ): cdk.CustomResource {
         const lambdaPolicyStatement = new iam.PolicyStatement({
             actions: ['servicecatalog:GetApplication'],
@@ -120,6 +123,8 @@ export class AppRegistry extends Construct implements cdk.IAspect {
         });
 
         const eventHandlerLambda = new NodejsFunction(this, 'event-handler-lambda', {
+            vpc: vpc,
+            vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS },
             entry: path.join(__dirname, '../lambdas/appRegistryWaiterLambda.ts'),
             handler: 'handler',
             description:

source/lib/constructs/baseInfrastructure.ts

@@ -41,6 +41,7 @@ export interface BaseInfrastructureProps {
     readonly vpcCidr?: string;
     readonly initBastion?: boolean;
     readonly initInstallationBucket?: boolean;
+    readonly selfManageInstallationBucketAssets?: boolean;
     readonly druidClusterName: string;
     readonly druidDeepStorageConfig?: DruidDeepStorageConfig;
     readonly oidcIdpConfig?: OidcIdpConfig;
@@ -122,54 +123,61 @@ export class BaseInfrastructure extends Construct {
                 }
             );
 
-            new BucketDeployment(this, 'bucket-deployment-scripts', {
-                sources: [Source.asset('lib/uploads/scripts')],
-                destinationBucket: this.installationBucket,
-                destinationKeyPrefix: SCRIPTS_FOLDER,
-            });
-
-            new BucketDeployment(this, 'bucket-deployment-extensions', {
-                sources: [Source.asset(`lib/docker/extensions`)],
-                destinationBucket: this.installationBucket,
-                destinationKeyPrefix: EXTENSIONS_FOLDER,
-                exclude: ['.gitkeep'],
-            });
-
-            new BucketDeployment(this, 'bucket-deployment-config', {
-                sources: [Source.asset('lib/uploads/config')],
-                destinationBucket: this.installationBucket,
-                destinationKeyPrefix: CONFIG_FOLDER,
-                exclude: ['*_version.txt'],
-            });
+            if (!props.selfManageInstallationBucketAssets) {
+                new BucketDeployment(this, 'bucket-deployment-scripts', {
+                    sources: [Source.asset('lib/uploads/scripts')],
+                    destinationBucket: this.installationBucket,
+                    destinationKeyPrefix: SCRIPTS_FOLDER,
+                    vpc: this.vpc,
+                });
 
-            new BucketDeployment(this, 'bucket-deployment-rds-ca-bundle', {
-                sources: [Source.asset('lib/docker/ca-certs')],
-                destinationBucket: this.installationBucket,
-                destinationKeyPrefix: 'ca-certs',
-            });
+                new BucketDeployment(this, 'bucket-deployment-extensions', {
+                    sources: [Source.asset(`lib/docker/extensions`)],
+                    destinationBucket: this.installationBucket,
+                    destinationKeyPrefix: EXTENSIONS_FOLDER,
+                    exclude: ['.gitkeep'],
+                    vpc: this.vpc,
+                });
 
-            this.druidImageDeployment = new BucketDeployment(
-                this,
-                'bucket-deployment-druid-image',
-                {
-                    sources: [Source.asset('druid-bin/')],
+                new BucketDeployment(this, 'bucket-deployment-config', {
+                    sources: [Source.asset('lib/uploads/config')],
                     destinationBucket: this.installationBucket,
-                    destinationKeyPrefix: DRUID_IMAGE_FOLDER,
-                    memoryLimit: 4096,
-                    useEfs: true,
+                    destinationKeyPrefix: CONFIG_FOLDER,
+                    exclude: ['*_version.txt'],
                     vpc: this.vpc,
-                }
-            );
+                });
 
-            this.zookeeperImageDeployment = new BucketDeployment(
-                this,
-                'bucket-deployment-zookeeper-image',
-                {
-                    sources: [Source.asset('zookeeper-bin/')],
+                new BucketDeployment(this, 'bucket-deployment-rds-ca-bundle', {
+                    sources: [Source.asset('lib/docker/ca-certs')],
                     destinationBucket: this.installationBucket,
-                    destinationKeyPrefix: ZOOKEEPER_IMAGE_FOLDER,
-                }
-            );
+                    destinationKeyPrefix: 'ca-certs',
+                    vpc: this.vpc,
+                });
+
+                this.druidImageDeployment = new BucketDeployment(
+                    this,
+                    'bucket-deployment-druid-image',
+                    {
+                        sources: [Source.asset('druid-bin/')],
+                        destinationBucket: this.installationBucket,
+                        destinationKeyPrefix: DRUID_IMAGE_FOLDER,
+                        memoryLimit: 4096,
+                        useEfs: true,
+                        vpc: this.vpc,
+                    }
+                );
+
+                this.zookeeperImageDeployment = new BucketDeployment(
+                    this,
+                    'bucket-deployment-zookeeper-image',
+                    {
+                        sources: [Source.asset('zookeeper-bin/')],
+                        destinationBucket: this.installationBucket,
+                        destinationKeyPrefix: ZOOKEEPER_IMAGE_FOLDER,
+                        vpc: this.vpc,
+                    }
+                );
+            }
         }
 
         if (props.initBastion) {

source/lib/constructs/configScheme.ts

@@ -275,6 +275,12 @@ export const configScheme = {
             $id: '#/properties/enableVulnerabilityScanJob',
             description: 'Whether to enable vulnerability scan job.',
         },
+        selfManageInstallationBucketAssets: {
+            type: 'boolean',
+            title: 'Enable self management for installation bucket assets',
+            $id: '#/properties/selfManageInstallationBucketAssets',
+            description: 'Whether to enable self management for installation bucket assets.',
+        },
         environmentAgnostic: {
             type: 'boolean',
             title: 'Environment Agnostic',

source/lib/constructs/internalCertificateAuthority.ts

@@ -17,6 +17,7 @@
 
 import * as cdk from 'aws-cdk-lib';
 import * as cr from 'aws-cdk-lib/custom-resources';
+import * as ec2 from 'aws-cdk-lib/aws-ec2';
 import * as kms from 'aws-cdk-lib/aws-kms';
 import * as lambda from 'aws-cdk-lib/aws-lambda';
 import * as lambdaNodejs from 'aws-cdk-lib/aws-lambda-nodejs';
@@ -25,10 +26,14 @@ import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
 
 import { Construct } from 'constructs';
 
+export interface InternalCertificateAuthorityProps {
+    vpc: ec2.IVpc;
+}
+
 export class InternalCertificateAuthority extends Construct {
   public readonly TlsCertificate: secretsmanager.ISecret;
 
-  public constructor(scope: Construct, id: string) {
+  public constructor(scope: Construct, id: string, props: InternalCertificateAuthorityProps) {
     super(scope, id);
 
     this.TlsCertificate = new secretsmanager.Secret(this, 'tls-certificate', {
@@ -44,6 +49,10 @@ export class InternalCertificateAuthority extends Construct {
       this,
       'tls-generator-handler',
       {
+        vpc: props.vpc,
+        vpcSubnets: {
+            subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS,
+        },
         entry: path.join(__dirname, '../lambdas/certificateGenerator.ts'),
         handler: 'onEventHandler',
         runtime: lambda.Runtime.NODEJS_LATEST,

source/lib/constructs/operationalMetricCollection.ts

@@ -15,6 +15,7 @@
 */
 import * as cdk from 'aws-cdk-lib';
 import * as cr from 'aws-cdk-lib/custom-resources';
+import * as ec2 from 'aws-cdk-lib/aws-ec2';
 import * as lambda from 'aws-cdk-lib/aws-lambda';
 import * as path from 'path';
 
@@ -24,6 +25,7 @@ import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs';
 import { addCfnNagSuppression } from './cfnNagSuppression';
 
 export interface OperationalMetricsCollectionProps {
+    vpc: ec2.IVpc;
     awsSolutionId: string;
     awsSolutionVersion: string;
     retainData: boolean;
@@ -44,6 +46,10 @@ export class OperationalMetricsCollection extends Construct {
         super(scope, id);
 
         const fn = new NodejsFunction(this, 'operational-metrics-handler', {
+            vpc: props.vpc,
+            vpcSubnets: {
+                subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS,
+            },
             entry: path.join(__dirname, './operationMetricCollectionLambda.ts'),
             handler: 'handler',
             description: 'Lambda for Operational Metrics collection',

source/lib/stacks/druidEc2Stack.ts

@@ -82,7 +82,10 @@ export class DruidEc2Stack extends DruidStack {
 
         const certificateGenerator = new InternalCertificateAuthority(
             this,
-            'internal-certificate-authority'
+            'internal-certificate-authority',
+            {
+                vpc: this.baseInfra.vpc,
+            }
         );
 
         const ec2Config = props.clusterParams.hostingConfig as Ec2Config;
@@ -197,9 +200,11 @@ export class DruidEc2Stack extends DruidStack {
         // create data tiers
         const dataAsgList = this.createDataTiers(asgContext);
         dataAsgList.forEach((dataAsg) => {
-            dataAsg.autoScalingGroup.node.addDependency(
-                this.baseInfra.druidImageDeployment!
-            );
+            if (this.baseInfra.druidImageDeployment) {
+                dataAsg.autoScalingGroup.node.addDependency(
+                    this.baseInfra.druidImageDeployment!,
+                );
+            }
             dataAsg.autoScalingGroup.node.addDependency(
                 zookeeper.zookeeperASGs[zookeeper.zookeeperASGs.length - 1]
             );
@@ -223,6 +228,7 @@ export class DruidEc2Stack extends DruidStack {
         });
 
         new OperationalMetricsCollection(this, 'metrics-collection', {
+            vpc: this.baseInfra.vpc,
             awsSolutionId: props.solutionId,
             awsSolutionVersion: props.solutionVersion,
             druidVersion: props.clusterParams.druidVersion,

source/lib/stacks/druidEksStack.ts

@@ -87,6 +87,7 @@ export class DruidEksStack extends DruidStack {
         }
 
         new OperationalMetricsCollection(this, 'metrics-collection', {
+            vpc: this.baseInfra.vpc,
             awsSolutionId: props.solutionId,
             awsSolutionVersion: props.solutionVersion,
             druidVersion: props.clusterParams.druidVersion,

source/lib/stacks/druidStack.ts

@@ -46,6 +46,7 @@ export abstract class DruidStack extends cdk.Stack {
             vpcCidr: props.vpcId ? undefined : props.vpcCidr, // don't set cidr for byo vpc
             initBastion: props.initBastion,
             initInstallationBucket: props.initInstallationBucket,
+            selfManageInstallationBucketAssets: props.selfManageInstallationBucketAssets,
             druidClusterName: props.clusterParams.druidClusterName,
             druidDeepStorageConfig: props.clusterParams.druidDeepStorageConfig,
             oidcIdpConfig: props.clusterParams.oidcIdpConfig,
@@ -111,6 +112,7 @@ export abstract class DruidStack extends cdk.Stack {
 
         cdk.Aspects.of(this).add(
             new AppRegistry(this, 'app-registry-aspect', {
+                vpc: this.baseInfra.vpc,
                 solutionId: props.solutionId,
                 solutionVersion: props.solutionVersion,
                 solutionName: props.solutionName,

source/lib/utils/types.ts

@@ -70,6 +70,7 @@ export interface DruidConfig {
     readonly retainData?: boolean;
     readonly enableVulnerabilityScanJob?: boolean;
     readonly environmentAgnostic?: boolean;
+    readonly selfManageInstallationBucketAssets?: boolean;
 
     readonly druidClusterName: string;
     readonly druidVersion: string;
@@ -98,6 +99,7 @@ export interface DruidStackProps extends cdk.StackProps {
     readonly vpcCidr?: string;
     readonly initBastion?: boolean;
     readonly initInstallationBucket?: boolean;
+    readonly selfManageInstallationBucketAssets?: boolean;
     readonly route53Params?: {
         readonly route53HostedZoneId: string;
         readonly route53HostedZoneName: string;