diff --git a/src/main/java/com/libertymutualgroup/herman/aws/cft/CftPush.java b/src/main/java/com/libertymutualgroup/herman/aws/cft/CftPush.java index ab7a792..b2a1708 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/cft/CftPush.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/cft/CftPush.java @@ -37,7 +37,6 @@ import com.amazonaws.services.lambda.AWSLambdaClientBuilder; import com.amazonaws.services.lambda.model.InvocationType; import com.amazonaws.services.lambda.model.InvokeRequest; -import com.amazonaws.util.IOUtils; import com.atlassian.bamboo.deployments.execution.DeploymentTaskContext; import com.atlassian.bamboo.task.TaskException; import com.atlassian.bamboo.variable.CustomVariableContext; @@ -48,22 +47,22 @@ import com.libertymutualgroup.herman.aws.ecs.TaskContextPropertyHandler; import com.libertymutualgroup.herman.logging.HermanLogger; import com.libertymutualgroup.herman.task.cft.CFTPushTaskProperties; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.libertymutualgroup.herman.util.FileUtil; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CftPush { @@ -75,6 +74,7 @@ public class CftPush { private static final String MAVEN_VERS = "maven.version"; private static final int RANDOM_PASSWORD_LENGTH = 20; private static final int POLLING_INTERVAL_MS = 10000; + private static final List CFT_FILE_NAMES = Arrays.asList("cft.template", "cft.yml", "cft.json"); private Properties props = new Properties(); private Properties output = new Properties(); private HermanLogger buildLogger; @@ -86,8 +86,8 @@ public class CftPush { private CFTPushTaskProperties taskProperties; public CftPush(HermanLogger buildLogger, DeploymentTaskContext taskContext, AWSCredentials sessionCredentials, - ClientConfiguration config, Regions region, CustomVariableContext customVariableContext, - CFTPushTaskProperties taskProperties) { + ClientConfiguration config, Regions region, CustomVariableContext customVariableContext, + CFTPushTaskProperties taskProperties) { this.buildLogger = buildLogger; this.taskContext = taskContext; @@ -112,7 +112,8 @@ public void push() throws TaskException { String projecName = taskContext.getDeploymentContext().getDeploymentProjectName(); if (!this.taskProperties.getCftPushVariableBrokerLambda().isEmpty()) { - buildLogger.addLogEntry("Getting CFT variables from Lambda: " + this.taskProperties.getCftPushVariableBrokerLambda()); + buildLogger.addLogEntry( + "Getting CFT variables from Lambda: " + this.taskProperties.getCftPushVariableBrokerLambda()); introspectEnvironment(); } injectBambooContext(); @@ -199,16 +200,7 @@ private void injectBambooContext() { } private void createStack(String name) { - String root = taskContext.getRootDirectory().getAbsolutePath(); - File file = new File(root + File.separator + "cft.template"); - - String template; - try { - template = IOUtils.toString(new FileInputStream(file)); - } catch (IOException e1) { - throw new AwsExecException(e1); - } - + String template = getTemplate(); List parameters = convertPropsToCftParams(template); String deployEnvironment = taskContext.getDeploymentContext().getEnvironmentName(); @@ -265,9 +257,28 @@ private void createStack(String name) { } + private String getTemplate() { + String root = taskContext.getRootDirectory().getAbsolutePath(); + FileUtil fileUtil = new FileUtil(root, buildLogger); + + String template = null; + for (String fileName: CFT_FILE_NAMES) { + boolean fileExists = fileUtil.fileExists(fileName); + if (fileExists) { + template = fileUtil.findFile(fileName, false); + buildLogger.addLogEntry("Template used: " + fileName); + } + } + if (template == null) { + throw new AwsExecException("CloudFormation template not found. Valid file names: " + + String.join(", ", CFT_FILE_NAMES)); + } + return template; + } + private List convertPropsToCftParams(String template) { List parameters = new ArrayList<>(); - for (Object key : props.keySet()) { + for (Object key: props.keySet()) { if (template.contains((String) key)) { parameters.add(new Parameter().withParameterKey((String) key) .withParameterValue(props.getProperty((String) key))); @@ -293,7 +304,7 @@ private void introspectEnvironment() throws TaskException { throw new TaskException(e.getMessage(), e); } - for (Map.Entry entry : variables.entrySet()) { + for (Map.Entry entry: variables.entrySet()) { buildLogger.addLogEntry("Injecting " + entry.getKey() + " = " + entry.getValue()); props.put(entry.getKey(), entry.getValue()); } @@ -309,14 +320,14 @@ private void outputStack(String stackName) { DescribeStackResourcesResult res = cftClient.describeStackResources(req); - for (StackResource r : res.getStackResources()) { + for (StackResource r: res.getStackResources()) { buildLogger.addLogEntry(r.getPhysicalResourceId()); buildLogger.addLogEntry(r.getResourceType()); output.put("aws.stack." + r.getLogicalResourceId(), r.getPhysicalResourceId()); } List resources = new ArrayList<>(); - for (StackResource r : res.getStackResources()) { + for (StackResource r: res.getStackResources()) { if ("AWS::ECS::TaskDefinition".equals(r.getResourceType())) { String id = r.getPhysicalResourceId(); String task = id.split("/")[1]; @@ -362,7 +373,7 @@ private void sleep() { } private Boolean reportStatusAndCheckCompletionOf(List stacks) { - for (Stack stack : stacks) { + for (Stack stack: stacks) { reportStatusOf(stack); if (stack.getStackStatus().contains("IN_PROGRESS")) { return false; diff --git a/src/main/java/com/libertymutualgroup/herman/aws/credentials/BambooCredentialsHandler.java b/src/main/java/com/libertymutualgroup/herman/aws/credentials/BambooCredentialsHandler.java index 6c1bc23..1e7227f 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/credentials/BambooCredentialsHandler.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/credentials/BambooCredentialsHandler.java @@ -26,6 +26,10 @@ public class BambooCredentialsHandler extends CredentialsHandler { + private BambooCredentialsHandler() { + throw new IllegalAccessError("Utility class"); + } + public static AWSCredentials getCredentials(CommonTaskContext context) { if (lookupVar("custom.aws.accessKeyId", context) != null) { return new BasicSessionCredentials(lookupVar("custom.aws.accessKeyId", context), diff --git a/src/main/java/com/libertymutualgroup/herman/aws/credentials/CredentialsHandler.java b/src/main/java/com/libertymutualgroup/herman/aws/credentials/CredentialsHandler.java index e567fdc..2dc880b 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/credentials/CredentialsHandler.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/credentials/CredentialsHandler.java @@ -20,6 +20,11 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; public class CredentialsHandler { + + CredentialsHandler() { + throw new IllegalAccessError("Utility class"); + } + public static AWSCredentials getCredentials() { return new DefaultAWSCredentialsProviderChain().getCredentials(); } diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/CliPropertyHandler.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/CliPropertyHandler.java index 3ff58c4..5412dd2 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/CliPropertyHandler.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/CliPropertyHandler.java @@ -40,13 +40,13 @@ public class CliPropertyHandler implements PropertyHandler { private Set propertyKeysUsed = new HashSet<>(); - private HermanLogger logger; + private HermanLogger hermanLogger; private String environmentName; private String rootDirectory; private Map customVariables; - public CliPropertyHandler(HermanLogger logger, String environmentName, String rootDirectory, Map customVariables) { - this.logger = logger; + public CliPropertyHandler(HermanLogger hermanLogger, String environmentName, String rootDirectory, Map customVariables) { + this.hermanLogger = hermanLogger; this.environmentName = environmentName; this.rootDirectory = rootDirectory; this.customVariables = customVariables; @@ -132,7 +132,7 @@ public String lookupVariable(String key) { } private void importPropFiles() { - FileUtil util = new FileUtil(this.rootDirectory, this.logger); + FileUtil util = new FileUtil(this.rootDirectory, this.hermanLogger); String envProps = util.findFile(this.environmentName + ".properties", true); if (props != null && envProps != null) { @@ -141,7 +141,7 @@ private void importPropFiles() { props.load(propStream); } catch (IOException e) { LOGGER.debug("Error loading properties file: " + this.environmentName, e); - this.logger.addLogEntry("Error loading " + this.environmentName + ".properties: " + e.getMessage()); + this.hermanLogger.addLogEntry("Error loading " + this.environmentName + ".properties: " + e.getMessage()); } } } diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/EcsPush.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/EcsPush.java index 8b62aeb..bdcc3f9 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/EcsPush.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/EcsPush.java @@ -268,7 +268,7 @@ public void push() { boolean useAlb = decider.shouldUseAlb(definition.getAppName(), definition); DnsRegistrar dnsRegistrar = new DnsRegistrar(lambdaClient, logger, taskProperties.getDnsBrokerLambda()); - CertHandler certHandler = new CertHandler(iamClient, logger, taskProperties.getSslCertificates()); + CertHandler certHandler = new CertHandler(logger, taskProperties.getSslCertificates()); if (useAlb) { EcsLoadBalancerV2Handler loadBalancerV2Handler = new EcsLoadBalancerV2Handler(elbV2Client, certHandler, dnsRegistrar, logger, taskProperties); diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/newrelic/NewRelicBroker.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/newrelic/NewRelicBroker.java index ff2f55b..55b52a8 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/newrelic/NewRelicBroker.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/newrelic/NewRelicBroker.java @@ -77,8 +77,7 @@ public void brokerNewRelicApplicationDeployment( InvokeResult invokeResult = this.lambdaClient.invoke(dnsBrokerInvokeRequest); - if (isSuccessful(invokeResult.getStatusCode()) && StringUtils - .isEmpty(invokeResult.getFunctionError())) { + if (isSuccessful(invokeResult.getStatusCode()) && StringUtils.isEmpty(invokeResult.getFunctionError())) { String nrBrokerResponseJson = new String(invokeResult.getPayload().array(), Charset.forName("UTF-8")); NewRelicBrokerResponse response; @@ -103,7 +102,8 @@ public void brokerNewRelicApplicationDeployment( addNewRelicLinkToLogs(response.getApplicationId()); } else { buildLogger.addLogEntry("... Error thrown by the NR Broker given payload: " + payload); - throw new RuntimeException("Error invoking the New Relic Broker: " + invokeResult); + String nrBrokerResponseJson = new String(invokeResult.getPayload().array(), Charset.forName("UTF-8")); + throw new RuntimeException("Error invoking the New Relic Broker: " + nrBrokerResponseJson); } } diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/s3/S3Broker.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/s3/S3Broker.java index b9e9d3d..0fd685c 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/s3/S3Broker.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/broker/s3/S3Broker.java @@ -74,9 +74,11 @@ public BucketMeta brokerFromConfigurationFile() { .withClientConfiguration(BambooCredentialsHandler.getConfiguration()).withRegion(context.getRegion()).build(); TagSet tags = new TagSet(); - tags.setTag(taskProperties.getSbuTagKey(), configuration.getSbu()); - tags.setTag(taskProperties.getOrgTagKey(), configuration.getOrg()); - tags.setTag(taskProperties.getAppTagKey(), configuration.getAppName()); + if (taskProperties != null) { + tags.setTag(taskProperties.getSbuTagKey(), configuration.getSbu()); + tags.setTag(taskProperties.getOrgTagKey(), configuration.getOrg()); + tags.setTag(taskProperties.getAppTagKey(), configuration.getAppName()); + } String policy = null; if (configuration.getPolicyName() != null) { @@ -96,10 +98,12 @@ public BucketMeta brokerFromConfigurationFile() { public void brokerBucketFromEcsPush(AmazonS3 client, S3Bucket bucket, String bucketPolicy, EcsClusterMetadata clusterMetadata, EcsPushDefinition definition) { TagSet tags = new TagSet(); - tags.setTag(taskProperties.getSbuTagKey(), clusterMetadata.getNewrelicSbuTag()); - tags.setTag(taskProperties.getOrgTagKey(), clusterMetadata.getNewrelicOrgTag()); - tags.setTag(taskProperties.getAppTagKey(), definition.getAppName()); - tags.setTag(taskProperties.getClusterTagKey(), clusterMetadata.getClusterId()); + if (taskProperties != null) { + tags.setTag(taskProperties.getSbuTagKey(), clusterMetadata.getNewrelicSbuTag()); + tags.setTag(taskProperties.getOrgTagKey(), clusterMetadata.getNewrelicOrgTag()); + tags.setTag(taskProperties.getAppTagKey(), definition.getAppName()); + tags.setTag(taskProperties.getClusterTagKey(), clusterMetadata.getClusterId()); + } S3InjectConfiguration configuration = new S3InjectConfiguration(); configuration.setAppName(bucket.getName()); configuration.setSbu(clusterMetadata.getNewrelicSbuTag()); diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterIntrospector.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterIntrospector.java index a237ab1..3f77d5d 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterIntrospector.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterIntrospector.java @@ -24,18 +24,12 @@ import com.amazonaws.services.cloudformation.model.StackResource; import com.amazonaws.services.cloudformation.model.Tag; import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest; -import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult; import com.amazonaws.services.ec2.model.DescribeSubnetsResult; import com.amazonaws.services.ec2.model.DescribeVpcsResult; -import com.amazonaws.services.ec2.model.Filter; -import com.amazonaws.services.ec2.model.SecurityGroup; import com.amazonaws.services.ec2.model.Subnet; import com.amazonaws.services.ec2.model.Vpc; import com.libertymutualgroup.herman.aws.AwsExecException; import com.libertymutualgroup.herman.logging.HermanLogger; - -import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -60,7 +54,7 @@ public EcsClusterMetadata introspect(String name) { DescribeStackResourcesResult clusterStackResult = cftClient.describeStackResources(req); - for (StackResource r : clusterStackResult.getStackResources()) { + for (StackResource r: clusterStackResult.getStackResources()) { updateClusterMetadataWithStackResourceValue(ecsClusterMetadata, r); } @@ -76,7 +70,7 @@ public EcsClusterMetadata introspect(String name) { DescribeVpcsResult res = ec2Client.describeVpcs(); Vpc vpc = null; - for (Vpc v : res.getVpcs()) { + for (Vpc v: res.getVpcs()) { if (isProperVpc(v)) { vpc = v; } @@ -90,9 +84,9 @@ public EcsClusterMetadata introspect(String name) { DescribeSubnetsResult sub = ec2Client.describeSubnets(); - for (Subnet net : sub.getSubnets()) { + for (Subnet net: sub.getSubnets()) { if (subnetMatches(vpc, net)) { - for (com.amazonaws.services.ec2.model.Tag t : net.getTags()) { + for (com.amazonaws.services.ec2.model.Tag t: net.getTags()) { if ("Name".equals(t.getKey())) { if (t.getValue().contains("private-elb")) { elbSubnets.add(net.getSubnetId()); @@ -103,7 +97,6 @@ public EcsClusterMetadata introspect(String name) { } } } - ecsClusterMetadata.setAkamaiSecurityGroup(getAkamaiSecurityGroups(vpc)); logger.addLogEntry("Introspection complete:"); logger.addLogEntry(ecsClusterMetadata.toString()); @@ -150,10 +143,12 @@ private void updateClusterMetadataWithStackResourceValue(EcsClusterMetadata ecsC } private boolean isProperVpc(Vpc vpc) { - String[] defaultIds = {"sandbox-vpc", "dev-vpc", "nonprod-vpc", "prod-vpc"}; + String[] defaultIds = {"sandbox", "dev", "nonprod", "prod"}; - for (com.amazonaws.services.ec2.model.Tag t : vpc.getTags()) { - if ("Name".equals(t.getKey()) && Arrays.asList(defaultIds).contains(t.getValue())) { + for (com.amazonaws.services.ec2.model.Tag t: vpc.getTags()) { + if ("Name".equals(t.getKey()) + && Arrays.asList(defaultIds).stream() + .filter(defaultId -> t.getValue().contains(defaultId)).findAny().isPresent()) { return true; } } @@ -163,42 +158,4 @@ private boolean isProperVpc(Vpc vpc) { private boolean subnetMatches(Vpc vpc, Subnet subnet) { return vpc != null && subnet != null && subnet.getVpcId() != null && subnet.getVpcId().equals(vpc.getVpcId()); } - - - private List getAkamaiSecurityGroups(Vpc vpc) { - String region = null; - for (com.amazonaws.services.ec2.model.Tag t : vpc.getTags()) { - if ("Name".equals(t.getKey())) { - String name = t.getValue(); - region = name.replaceAll("-vpc", ""); - } - } - - List groups = new ArrayList<>(); - String prefix = "aws-shared-external-elb-" + region; - SecurityGroup groupOne = getSecurityGroup(prefix + "-1"); - SecurityGroup groupTwo = getSecurityGroup(prefix + "-2"); - groups.add(groupOne.getGroupId()); - groups.add(groupTwo.getGroupId()); - - return groups; - } - - - private SecurityGroup getSecurityGroup(String sgName) { - Filter filter = new Filter().withName("tag:Name").withValues(sgName); - DescribeSecurityGroupsRequest secReq = new DescribeSecurityGroupsRequest().withFilters(filter); - - DescribeSecurityGroupsResult sgResult = ec2Client.describeSecurityGroups(secReq); - - SecurityGroup secGroup; - if (sgResult.getSecurityGroups().size() == 1) { - secGroup = sgResult.getSecurityGroups().get(0); - } else { - logger.addLogEntry("Used: " + sgName); - - throw new AwsExecException("Error looking up SG :" + sgResult.getSecurityGroups().size()); - } - return secGroup; - } } diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterMetadata.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterMetadata.java index 5057a6a..4c26ffb 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterMetadata.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/cluster/EcsClusterMetadata.java @@ -24,7 +24,6 @@ public class EcsClusterMetadata { private String rdsSecurityGroup; private String appSecurityGroup; private List elbSecurityGroups = new ArrayList<>(); - private List akamaiSecurityGroup; private String clusterId; private List elbSubnets = new ArrayList<>(); private List publicSubnets = new ArrayList<>(); @@ -63,14 +62,6 @@ public void setElbSecurityGroups(List elbSecurityGroups) { this.elbSecurityGroups = elbSecurityGroups; } - public List getAkamaiSecurityGroup() { - return akamaiSecurityGroup; - } - - public void setAkamaiSecurityGroup(List akamaiSecurityGroup) { - this.akamaiSecurityGroup = akamaiSecurityGroup; - } - public String getClusterId() { return clusterId; } @@ -181,7 +172,6 @@ public String toString() { "rdsSecurityGroup='" + rdsSecurityGroup + '\'' + ", appSecurityGroup='" + appSecurityGroup + '\'' + ", elbSecurityGroups=" + elbSecurityGroups + - ", akamaiSecurityGroup=" + akamaiSecurityGroup + ", clusterId='" + clusterId + '\'' + ", elbSubnets=" + elbSubnets + ", publicSubnets=" + publicSubnets + diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/CertHandler.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/CertHandler.java index bee1df8..dbf9d2b 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/CertHandler.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/CertHandler.java @@ -15,9 +15,6 @@ */ package com.libertymutualgroup.herman.aws.ecs.loadbalancing; -import com.amazonaws.services.identitymanagement.AmazonIdentityManagement; -import com.amazonaws.services.identitymanagement.model.ListServerCertificatesResult; -import com.amazonaws.services.identitymanagement.model.ServerCertificateMetadata; import com.libertymutualgroup.herman.aws.AwsExecException; import com.libertymutualgroup.herman.logging.HermanLogger; @@ -27,34 +24,21 @@ public class CertHandler { private static final String HTTPS = "HTTPS"; - - private final AmazonIdentityManagement iamClient; private final HermanLogger buildLogger; private final List sslCertificates; - public CertHandler(AmazonIdentityManagement iamClient, HermanLogger buildLogger, - List sslCertificates) { - this.iamClient = iamClient; + public CertHandler(HermanLogger buildLogger, List sslCertificates) { this.buildLogger = buildLogger; this.sslCertificates = sslCertificates; } - public DeriveCertResult deriveCert(String protocol, String urlSuffix, String urlPrefix) { - DeriveCertResult deriveCertResult = new DeriveCertResult(); + public SSLCertificate deriveCert(String protocol, String urlSuffix, String urlPrefix) { + SSLCertificate sslCertificate = null; if (HTTPS.equals(protocol)) { - SSLCertificate sslCertificate = getSSLCertificateByUrl(urlPrefix, urlSuffix); - deriveCertResult.setSslCertificate(sslCertificate); - - ListServerCertificatesResult certResult = iamClient.listServerCertificates(); - for (ServerCertificateMetadata meta : certResult.getServerCertificateMetadataList()) { - if (meta.getArn().endsWith(sslCertificate.getPathSuffix())) { - deriveCertResult.setCertArn(meta.getArn()); - break; - } - } - buildLogger.addLogEntry("SSL cert found: " + deriveCertResult.getCertArn()); + sslCertificate = getSSLCertificateByUrl(urlPrefix, urlSuffix); + buildLogger.addLogEntry("SSL cert found: " + sslCertificate.getArn()); } - return deriveCertResult; + return sslCertificate; } public SSLCertificate getSSLCertificateByUrl(String prefix, String suffix) { diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DeriveCertResult.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DeriveCertResult.java deleted file mode 100644 index 826a75c..0000000 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DeriveCertResult.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.libertymutualgroup.herman.aws.ecs.loadbalancing; - -public class DeriveCertResult { - - private SSLCertificate sslCertificate; - private String certArn; - - public SSLCertificate getSslCertificate() { - return sslCertificate; - } - - public void setSslCertificate(SSLCertificate sslCertificate) { - this.sslCertificate = sslCertificate; - } - - public String getCertArn() { - return certArn; - } - - public void setCertArn(String certArn) { - this.certArn = certArn; - } - - public DeriveCertResult withSslCertificate(final SSLCertificate sslCertificate) { - this.sslCertificate = sslCertificate; - return this; - } - - public DeriveCertResult withCertArn(final String certArn) { - this.certArn = certArn; - return this; - } - - @Override - public String toString() { - return "DeriveCertResult{" + - "sslCertificate=" + sslCertificate + - ", certArn='" + certArn + '\'' + - '}'; - } -} \ No newline at end of file diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsBrokerRequest.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsBrokerRequest.java index 00043d5..aec354e 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsBrokerRequest.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsBrokerRequest.java @@ -1,74 +1,71 @@ package com.libertymutualgroup.herman.aws.ecs.loadbalancing; -import com.amazonaws.services.cloudformation.model.Tag; -import java.util.List; - public class DnsBrokerRequest { + private String elbName; + private String elbType; + private String protocol; private String vanityUrl; - private String awsUrl; - private String appName; - private List tags; - public String getVanityUrl() { - return vanityUrl; + public String getElbName() { + return elbName; } - public void setVanityUrl(String vanityUrl) { - this.vanityUrl = vanityUrl; + public void setElbName(String elbName) { + this.elbName = elbName; } - public String getAwsUrl() { - return awsUrl; + public String getElbType() { + return elbType; } - public void setAwsUrl(String awsUrl) { - this.awsUrl = awsUrl; + public void setElbType(String elbType) { + this.elbType = elbType; } - public String getAppName() { - return appName; + public String getProtocol() { + return protocol; } - public void setAppName(String appName) { - this.appName = appName; + public void setProtocol(String protocol) { + this.protocol = protocol; } - public List getTags() { - return tags; + public String getVanityUrl() { + return vanityUrl; } - public void setTags(List tags) { - this.tags = tags; + public void setVanityUrl(String vanityUrl) { + this.vanityUrl = vanityUrl; } - public DnsBrokerRequest withVanityUrl(final String vanityUrl) { - this.vanityUrl = vanityUrl; + public DnsBrokerRequest withElbName(final String elbName) { + this.elbName = elbName; return this; } - public DnsBrokerRequest withAwsUrl(final String awsUrl) { - this.awsUrl = awsUrl; + public DnsBrokerRequest withElbType(final String elbType) { + this.elbType = elbType; return this; } - public DnsBrokerRequest withAppName(final String appName) { - this.appName = appName; + public DnsBrokerRequest withProtocol(final String protocol) { + this.protocol = protocol; return this; } - public DnsBrokerRequest withTags(final List tags) { - this.tags = tags; + public DnsBrokerRequest withVanityUrl(final String vanityUrl) { + this.vanityUrl = vanityUrl; return this; } @Override public String toString() { return "DnsBrokerRequest{" + - "vanityUrl='" + vanityUrl + '\'' + - ", awsUrl='" + awsUrl + '\'' + - ", appName='" + appName + '\'' + - ", tags=" + tags + + "elbName='" + elbName + '\'' + + ", elbType='" + elbType + '\'' + + ", protocol='" + protocol + '\'' + + ", vanityUrl='" + vanityUrl + '\'' + '}'; } } diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsRegistrar.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsRegistrar.java index 385c7d3..371eb07 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsRegistrar.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/DnsRegistrar.java @@ -15,15 +15,13 @@ */ package com.libertymutualgroup.herman.aws.ecs.loadbalancing; -import com.amazonaws.services.cloudformation.model.Tag; import com.amazonaws.services.lambda.AWSLambda; import com.amazonaws.services.lambda.model.InvocationType; import com.amazonaws.services.lambda.model.InvokeRequest; import com.amazonaws.services.lambda.model.InvokeResult; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.libertymutualgroup.herman.aws.ecs.broker.domain.HermanBrokerStatus; -import com.libertymutualgroup.herman.aws.ecs.broker.domain.HermanBrokerUpdate; +import com.libertymutualgroup.herman.aws.AwsExecException; import com.libertymutualgroup.herman.logging.HermanLogger; import org.apache.commons.lang3.StringUtils; @@ -44,51 +42,47 @@ public DnsRegistrar(AWSLambda lambdaClient, HermanLogger buildLogger, String dns this.dnsLambda = dnsLambda; } - public void registerDns(String vanityUrl, String awsUrl, String appName, List tags) { - String payload = getPayload(vanityUrl, awsUrl, appName, tags); - InvokeRequest dnsBrokerInvokeRequest = new InvokeRequest() - .withFunctionName(dnsLambda) - .withInvocationType(InvocationType.RequestResponse) - .withPayload(payload); + public void registerDns(String elbName, String elbType, String protocol, String registeredUrl) { + DnsBrokerRequest dnsBrokerRequest = new DnsBrokerRequest() + .withElbName(elbName) + .withElbType(elbType) + .withProtocol(protocol) + .withVanityUrl(registeredUrl); buildLogger.addLogEntry("... Invoke request sent to the DNS Broker"); + InvokeResult invokeResult = this.lambdaClient.invoke(getInvokeRequest(dnsBrokerRequest)); - InvokeResult invokeResult = this.lambdaClient.invoke(dnsBrokerInvokeRequest); - if (isSuccessful(invokeResult.getStatusCode()) && StringUtils - .isEmpty(invokeResult.getFunctionError())) { + if (isSuccessful(invokeResult.getStatusCode()) && StringUtils.isEmpty(invokeResult.getFunctionError())) { String dnsBrokerUpdatesJson = new String(invokeResult.getPayload().array(), Charset.forName("UTF-8")); - List updates; + List updates; try { updates = new ObjectMapper() - .readValue(dnsBrokerUpdatesJson, new TypeReference>() {}); + .readValue(dnsBrokerUpdatesJson, new TypeReference>() {}); } catch (Exception e) { throw new RuntimeException("Unable to parse broker updates from: " + dnsBrokerUpdatesJson, e); } - updates.stream().forEach(update -> { + updates.stream().forEach(update -> buildLogger .addLogEntry( - String.format("... DNS Broker: [%s] %s", update.getStatus(), update.getMessage())); - - if (HermanBrokerStatus.ERROR.equals(update.getStatus())) { - throw new RuntimeException("DNS registration error"); - } - }); + String.format("... DNS Broker: %s", update)) + ); } else { - buildLogger.addLogEntry("... Error thrown by the DNS Broker given payload: " + payload); - throw new RuntimeException("Error invoking DNS Broker: " + invokeResult); + buildLogger.addLogEntry("... Error thrown by the DNS Broker given payload: " + dnsBrokerRequest); + String brokerResponseJson = new String(invokeResult.getPayload().array(), Charset.forName("UTF-8")); + throw new RuntimeException("Error invoking DNS Broker: " + brokerResponseJson); } } - private String getPayload(String vanityUrl, String awsUrl, String appName, List tags) { + private InvokeRequest getInvokeRequest(DnsBrokerRequest dnsBrokerRequest) { + InvokeRequest dnsBrokerInvokeRequest; try { - DnsBrokerRequest dnsBrokerRequest = new DnsBrokerRequest() - .withAppName(appName) - .withAwsUrl(awsUrl) - .withVanityUrl(vanityUrl) - .withTags(tags); - return new ObjectMapper().writeValueAsString(dnsBrokerRequest); + dnsBrokerInvokeRequest = new InvokeRequest() + .withFunctionName(dnsLambda) + .withInvocationType(InvocationType.RequestResponse) + .withPayload(new ObjectMapper().writeValueAsString(dnsBrokerRequest)); } catch (Exception ex) { - throw new RuntimeException("Error getting DNS broker payload", ex); + throw new AwsExecException("Error building invoke request", ex); } + return dnsBrokerInvokeRequest; } } diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandler.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandler.java index c1db19e..0d2f9d9 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandler.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandler.java @@ -28,24 +28,23 @@ import com.amazonaws.services.elasticloadbalancing.model.CreateLoadBalancerRequest; import com.amazonaws.services.elasticloadbalancing.model.DeleteLoadBalancerListenersRequest; import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersRequest; -import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersResult; import com.amazonaws.services.elasticloadbalancing.model.DuplicateLoadBalancerNameException; import com.amazonaws.services.elasticloadbalancing.model.HealthCheck; import com.amazonaws.services.elasticloadbalancing.model.Listener; import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerDescription; import com.amazonaws.services.elasticloadbalancing.model.SetLoadBalancerPoliciesOfListenerRequest; +import com.libertymutualgroup.herman.aws.AwsExecException; import com.libertymutualgroup.herman.aws.ecs.EcsPortHandler; import com.libertymutualgroup.herman.aws.ecs.EcsPushDefinition; import com.libertymutualgroup.herman.aws.ecs.cluster.EcsClusterMetadata; import com.libertymutualgroup.herman.logging.HermanLogger; -import com.libertymutualgroup.herman.task.common.CommonTaskProperties; -import org.apache.commons.collections4.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.libertymutualgroup.herman.task.ecs.ECSPushTaskProperties; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EcsLoadBalancerHandler { @@ -56,10 +55,10 @@ public class EcsLoadBalancerHandler { private CertHandler certHandler; private HermanLogger buildLogger; private DnsRegistrar dnsRegistrar; - private CommonTaskProperties taskProperties; + private ECSPushTaskProperties taskProperties; public EcsLoadBalancerHandler(AmazonElasticLoadBalancing elbClient, CertHandler certHandler, - DnsRegistrar dnsRegistrar, HermanLogger buildLogger, CommonTaskProperties taskProperties) { + DnsRegistrar dnsRegistrar, HermanLogger buildLogger, ECSPushTaskProperties taskProperties) { this.elbClient = elbClient; this.certHandler = certHandler; this.dnsRegistrar = dnsRegistrar; @@ -84,37 +83,34 @@ public LoadBalancer createLoadBalancer(EcsClusterMetadata clusterMetadata, EcsPu } String urlSuffix = definition.getService().getUrlSuffix(); - DeriveCertResult deriveCertResult = certHandler.deriveCert(protocol, urlSuffix, urlPrefix); + SSLCertificate sslCertificate = certHandler.deriveCert(protocol, urlSuffix, urlPrefix); ContainerDefinition webContainer = portHandler.findContainerWithExposedPort(definition, false); Integer randomPort = webContainer.getPortMappings().get(0).getHostPort(); Integer containerPort = webContainer.getPortMappings().get(0).getContainerPort(); String containerName = webContainer.getName(); - boolean isInternetFacingUrlScheme = certHandler.isInternetFacingUrlScheme(deriveCertResult.getSslCertificate(), + boolean isInternetFacingUrlScheme = certHandler.isInternetFacingUrlScheme(sslCertificate, definition.getService().getUrlSchemeOverride()); - boolean isUsingInternalSubnets = true; String elbScheme; List elbSubnets; if (isInternetFacingUrlScheme || "internet-facing".equals(definition.getService().getElbSchemeOverride())) { elbScheme = "internet-facing"; - isUsingInternalSubnets = false; elbSubnets = clusterMetadata.getPublicSubnets(); } else { elbScheme = "internal"; elbSubnets = clusterMetadata.getElbSubnets(); } - // Handle akamai List elbSecurityGroups = new ArrayList<>(); - if (isInternetFacingUrlScheme && HTTPS.equals(protocol)) { - elbSecurityGroups.addAll(clusterMetadata.getAkamaiSecurityGroup()); + if ("internet-facing".equals(elbScheme) && HTTPS.equals(protocol)) { + elbSecurityGroups.addAll(taskProperties.getExternalElbSecurityGroups()); } else { elbSecurityGroups.addAll(clusterMetadata.getElbSecurityGroups()); } List listeners = generateListeners(definition.getService().getElbSourcePorts(), randomPort, protocol, - deriveCertResult.getCertArn()); + sslCertificate.getArn()); List tags = getElbTagList( clusterMetadata.getClusterCftStackTags(), appName); CreateLoadBalancerRequest createLoadBalancerRequest = new CreateLoadBalancerRequest().withSubnets(elbSubnets) @@ -124,21 +120,36 @@ public LoadBalancer createLoadBalancer(EcsClusterMetadata clusterMetadata, EcsPu elbClient.createLoadBalancer(createLoadBalancerRequest); } catch (DuplicateLoadBalancerNameException e) { LOGGER.debug("Error creating ELB: " + appName, e); + + LoadBalancerDescription loadBalancerDescription = elbClient.describeLoadBalancers( + new DescribeLoadBalancersRequest().withLoadBalancerNames(appName)) + .getLoadBalancerDescriptions().get(0); + if (!elbScheme.equals(loadBalancerDescription.getScheme())) { + throw new AwsExecException(String.format("Expected scheme (%s) and actual scheme (%s) do not match. " + + "Load balancer %s must be re-created.", elbScheme, loadBalancerDescription.getScheme(), appName)); + } + buildLogger.addLogEntry("Updating ELB: " + appName); elbClient.deleteLoadBalancerListeners(new DeleteLoadBalancerListenersRequest().withLoadBalancerName(appName) .withLoadBalancerPorts(CollectionUtils.isNotEmpty(definition.getService().getElbSourcePorts()) ? definition.getService().getElbSourcePorts() : Arrays.asList(443))); + elbClient.createLoadBalancerListeners( new CreateLoadBalancerListenersRequest().withListeners(listeners).withLoadBalancerName(appName)); + + buildLogger.addLogEntry("... Resetting security groups: " + String.join(", ", elbSecurityGroups)); elbClient.applySecurityGroupsToLoadBalancer(new ApplySecurityGroupsToLoadBalancerRequest() .withLoadBalancerName(appName).withSecurityGroups(elbSecurityGroups)); - elbClient.addTags(new AddTagsRequest().withLoadBalancerNames(appName).withTags(tags)); + buildLogger.addLogEntry("... Resetting subnets: " + String.join(", ", elbSubnets)); elbClient.attachLoadBalancerToSubnets( new AttachLoadBalancerToSubnetsRequest().withSubnets(elbSubnets).withLoadBalancerName(appName)); + + buildLogger.addLogEntry("... Resetting tags"); + elbClient.addTags(new AddTagsRequest().withLoadBalancerNames(appName).withTags(tags)); } catch (Exception ex) { - throw new RuntimeException("Error creating ELB: " + createLoadBalancerRequest); + throw new RuntimeException("Error creating ELB: " + createLoadBalancerRequest, ex); } if (definition.getService().getAppStickinessCookie() != null) { @@ -175,21 +186,8 @@ public LoadBalancer createLoadBalancer(EcsClusterMetadata clusterMetadata, EcsPu elbClient.configureHealthCheck( new ConfigureHealthCheckRequest().withLoadBalancerName(appName).withHealthCheck(healthCheck)); - DescribeLoadBalancersResult elbResult = elbClient - .describeLoadBalancers(new DescribeLoadBalancersRequest().withLoadBalancerNames(appName)); - LoadBalancerDescription elbDesc = elbResult.getLoadBalancerDescriptions().get(0); - String registeredUrl = urlPrefix + "." + definition.getService().getUrlSuffix(); - - if (isUsingInternalSubnets) { - dnsRegistrar.registerDns(registeredUrl, elbDesc.getDNSName(), appName, - clusterMetadata.getClusterCftStackTags()); - buildLogger.addLogEntry("... URL Registered: " + protocol.toLowerCase() + "://" + registeredUrl); - } else { - buildLogger.addLogEntry( - "... Raw ELB DNS for Akamai: " + protocol.toLowerCase() + "://" + elbDesc.getDNSName()); - buildLogger.addLogEntry("... Expected Akamai url: " + protocol.toLowerCase() + "://" + registeredUrl); - } + dnsRegistrar.registerDns(appName, "classic", protocol, registeredUrl); buildLogger.addLogEntry("... ELB updates complete"); return new LoadBalancer().withContainerName(containerName).withContainerPort(containerPort) diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2Handler.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2Handler.java index fc33f3a..4893d32 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2Handler.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2Handler.java @@ -49,12 +49,11 @@ import com.libertymutualgroup.herman.aws.ecs.EcsPushDefinition; import com.libertymutualgroup.herman.aws.ecs.cluster.EcsClusterMetadata; import com.libertymutualgroup.herman.logging.HermanLogger; -import com.libertymutualgroup.herman.task.common.CommonTaskProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.libertymutualgroup.herman.task.ecs.ECSPushTaskProperties; import java.util.ArrayList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EcsLoadBalancerV2Handler { @@ -65,10 +64,10 @@ public class EcsLoadBalancerV2Handler { private CertHandler certHandler; private HermanLogger buildLogger; private DnsRegistrar dnsRegistrar; - private CommonTaskProperties taskProperties; + private ECSPushTaskProperties taskProperties; public EcsLoadBalancerV2Handler(AmazonElasticLoadBalancing elbClient, CertHandler certHandler, - DnsRegistrar dnsRegistrar, HermanLogger buildLogger, CommonTaskProperties taskProperties) { + DnsRegistrar dnsRegistrar, HermanLogger buildLogger, ECSPushTaskProperties taskProperties) { this.elbClient = elbClient; this.certHandler = certHandler; this.buildLogger = buildLogger; @@ -92,31 +91,28 @@ public LoadBalancer createLoadBalancer(EcsClusterMetadata clusterMetadata, EcsPu } String urlSuffix = definition.getService().getUrlSuffix(); - DeriveCertResult deriveCertResult = certHandler.deriveCert(protocol, urlSuffix, urlPrefix); + SSLCertificate sslCertificate = certHandler.deriveCert(protocol, urlSuffix, urlPrefix); ContainerDefinition webContainer = portHandler.findContainerWithExposedPort(definition, true); Integer containerPort = webContainer.getPortMappings().get(0).getContainerPort(); String containerName = webContainer.getName(); // Set scheme and subnets - boolean isInternetFacingUrlScheme = certHandler.isInternetFacingUrlScheme(deriveCertResult.getSslCertificate(), + boolean isInternetFacingUrlScheme = certHandler.isInternetFacingUrlScheme(sslCertificate, definition.getService().getUrlSchemeOverride()); - boolean isUsingInternalSubnets = true; String elbScheme; List elbSubnets; if (isInternetFacingUrlScheme || "internet-facing".equals(definition.getService().getElbSchemeOverride())) { elbScheme = "internet-facing"; - isUsingInternalSubnets = false; elbSubnets = clusterMetadata.getPublicSubnets(); } else { elbScheme = "internal"; elbSubnets = clusterMetadata.getElbSubnets(); } - // Set security groups - Akamai groups set for internet-facing apps List elbSecurityGroups = new ArrayList<>(); - if (isInternetFacingUrlScheme && HTTPS.equals(protocol)) { - elbSecurityGroups.addAll(clusterMetadata.getAkamaiSecurityGroup()); + if ("internet-facing".equals(elbScheme) && HTTPS.equals(protocol)) { + elbSecurityGroups.addAll(taskProperties.getExternalElbSecurityGroups()); } else { elbSecurityGroups.addAll(clusterMetadata.getElbSecurityGroups()); } @@ -157,24 +153,29 @@ public LoadBalancer createLoadBalancer(EcsClusterMetadata clusterMetadata, EcsPu loadBalancer = res.getLoadBalancers().get(0); elbClient.createListener(new CreateListenerRequest().withLoadBalancerArn(loadBalancer.getLoadBalancerArn()) - .withCertificates(new Certificate().withCertificateArn(deriveCertResult.getCertArn())) + .withCertificates(new Certificate().withCertificateArn(sslCertificate.getArn())) .withProtocol(ProtocolEnum.HTTPS) .withPort(443).withDefaultActions( new Action().withTargetGroupArn(grp.getTargetGroupArn()).withType(ActionTypeEnum.Forward))); waitForALBCreate(loadBalancer.getLoadBalancerArn()); } else { + if (!elbScheme.equals(loadBalancer.getScheme())) { + throw new AwsExecException(String.format("Expected scheme (%s) and actual scheme (%s) do not match. " + + "Load balancer %s must be re-created.", elbScheme, loadBalancer.getScheme(), appName)); + } + String arn = loadBalancer.getLoadBalancerArn(); buildLogger.addLogEntry("Updating existing ALB: " + arn); - buildLogger.addLogEntry("... Reset SG"); + buildLogger.addLogEntry("... Resetting security groups: " + String.join(", ", elbSecurityGroups)); elbClient.setSecurityGroups( new SetSecurityGroupsRequest().withLoadBalancerArn(arn).withSecurityGroups(elbSecurityGroups)); - buildLogger.addLogEntry("... Reset subnets"); + buildLogger.addLogEntry("... Resetting subnets: " + String.join(", ", elbSubnets)); elbClient.setSubnets(new SetSubnetsRequest().withLoadBalancerArn(arn).withSubnets(elbSubnets)); - buildLogger.addLogEntry("... Reset tags"); + buildLogger.addLogEntry("... Resetting tags"); elbClient.addTags(new AddTagsRequest().withResourceArns(arn) .withTags(tags)); @@ -193,7 +194,7 @@ public LoadBalancer createLoadBalancer(EcsClusterMetadata clusterMetadata, EcsPu Listener lis = elbClient.describeListeners(new DescribeListenersRequest().withLoadBalancerArn(arn)) .getListeners().get(0); elbClient.modifyListener(new ModifyListenerRequest().withListenerArn(lis.getListenerArn()) - .withCertificates(new Certificate().withCertificateArn(deriveCertResult.getCertArn()))); + .withCertificates(new Certificate().withCertificateArn(sslCertificate.getArn()))); } modifyTargetGroupAttributes(definition, grp); @@ -201,15 +202,7 @@ public LoadBalancer createLoadBalancer(EcsClusterMetadata clusterMetadata, EcsPu buildLogger.addLogEntry("Updating DNS registration"); String registeredUrl = urlPrefix + "." + definition.getService().getUrlSuffix(); - if (isUsingInternalSubnets) { - dnsRegistrar.registerDns(registeredUrl, loadBalancer.getDNSName(), appName, - clusterMetadata.getClusterCftStackTags()); - buildLogger.addLogEntry("... URL Registered: " + protocol.toLowerCase() + "://" + registeredUrl); - } else { - buildLogger.addLogEntry( - "... Raw ELB DNS for Akamai: " + protocol.toLowerCase() + "://" + loadBalancer.getDNSName()); - buildLogger.addLogEntry("... Expected Akamai url: " + protocol.toLowerCase() + "://" + registeredUrl); - } + dnsRegistrar.registerDns(appName, "application", protocol, registeredUrl); buildLogger.addLogEntry("ALB updates complete: " + loadBalancer.getLoadBalancerArn()); return new LoadBalancer().withContainerName(containerName).withContainerPort(containerPort) @@ -280,7 +273,7 @@ private com.amazonaws.services.elasticloadbalancingv2.model.LoadBalancer findLoa private List getElbTagList(List tags, String appName) { List result = new ArrayList<>(); - for (Tag cftTag : tags) { + for (Tag cftTag: tags) { if ("Name".equals(cftTag.getKey())) { result.add(new com.amazonaws.services.elasticloadbalancingv2.model.Tag() .withKey(this.taskProperties.getClusterTagKey()) @@ -319,9 +312,6 @@ private void waitForALBCreate(String arn) { buildLogger.addLogEntry("Interrupted while polling"); throw new AwsExecException("Interrupted while polling"); } - - } } - } \ No newline at end of file diff --git a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/SSLCertificate.java b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/SSLCertificate.java index 69963c7..5020a5e 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/SSLCertificate.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/SSLCertificate.java @@ -15,11 +15,14 @@ */ package com.libertymutualgroup.herman.aws.ecs.loadbalancing; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) public class SSLCertificate { private String urlSuffix; private String urlPrefix; - private String pathSuffix; + private String arn; private boolean internetFacingUrl; public String getUrlSuffix() { @@ -38,12 +41,12 @@ public void setUrlPrefix(String urlPrefix) { this.urlPrefix = urlPrefix; } - public String getPathSuffix() { - return pathSuffix; + public String getArn() { + return arn; } - public void setPathSuffix(String pathSuffix) { - this.pathSuffix = pathSuffix; + public void setArn(String arn) { + this.arn = arn; } public boolean isInternetFacingUrl() { @@ -64,13 +67,13 @@ public SSLCertificate withUrlPrefix(final String urlPrefix) { return this; } - public SSLCertificate withPathSuffix(final String pathSuffix) { - this.pathSuffix = pathSuffix; + public SSLCertificate withInternetFacingUrl(final boolean internetFacingUrl) { + this.internetFacingUrl = internetFacingUrl; return this; } - public SSLCertificate withInternetFacingUrl(final boolean internetFacingUrl) { - this.internetFacingUrl = internetFacingUrl; + public SSLCertificate withArn(final String arn) { + this.arn = arn; return this; } @@ -79,8 +82,8 @@ public String toString() { return "SSLCertificate{" + "urlSuffix='" + urlSuffix + '\'' + ", urlPrefix='" + urlPrefix + '\'' + - ", pathSuffix='" + pathSuffix + '\'' + + ", arn='" + arn + '\'' + ", internetFacingUrl=" + internetFacingUrl + '}'; } -} +} \ No newline at end of file diff --git a/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaBroker.java b/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaBroker.java index f97f9a0..c4b434f 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaBroker.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaBroker.java @@ -19,6 +19,15 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.regions.Regions; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; +import com.amazonaws.services.ec2.model.AmazonEC2Exception; +import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest; +import com.amazonaws.services.ec2.model.CreateSecurityGroupResult; +import com.amazonaws.services.ec2.model.CreateTagsRequest; +import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest; +import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult; +import com.amazonaws.services.ec2.model.Filter; import com.amazonaws.services.ecs.model.KeyValuePair; import com.amazonaws.services.identitymanagement.AmazonIdentityManagement; import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClientBuilder; @@ -71,6 +80,7 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -96,6 +106,7 @@ public class LambdaBroker { private AWSLambda lambdaClient; private AWSKMS kmsClient; private AmazonIdentityManagement iamClient; + private AmazonEC2 ec2Client; private CommonTaskProperties taskProperties; private AWSCredentials credentials; private Regions region; @@ -130,6 +141,12 @@ public LambdaBroker(LambdaPushContext context, HermanLogger buildLogger, Regions .withClientConfiguration(config) .withRegion(region) .build(); + + this.ec2Client = AmazonEC2ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withClientConfiguration(config) + .withRegion(region) + .build(); } public void brokerLambda() throws IOException { @@ -140,6 +157,9 @@ public void brokerLambda() throws IOException { new Tag().withTagKey(this.taskProperties.getAppTagKey()).withTagValue(this.configuration.getAppName())); Map tagMap = tags.stream().collect(Collectors.toMap(Tag::getTagKey, Tag::getTagValue)); + // Create/Update custom security group if needed + String customSecurityGroupId = getCustomSecurityGroupId(tagMap); + String functionArn; try { GetFunctionConfigurationResult describeResult = lambdaClient.getFunctionConfiguration( @@ -185,26 +205,9 @@ public void brokerLambda() throws IOException { throw new AwsExecException(ex); } - VpcConfig vpcConfig = new VpcConfig(); - if (this.configuration.getSubnetIds() != null) { - vpcConfig.setSubnetIds(this.configuration.getSubnetIds()); - } - if (this.configuration.getSecurityGroupIds() != null) { - vpcConfig.setSecurityGroupIds(this.configuration.getSecurityGroupIds()); - } - - HashMap environmentMap = new HashMap<>(); - if (this.configuration.getEnvironment() != null) { - for (KeyValuePair envVar : this.configuration.getEnvironment()) { - environmentMap.put(envVar.getName(), envVar.getValue()); - } - } - Environment environment = new Environment().withVariables(environmentMap); - - this.buildLogger.addLogEntry("... VPC Configuration: " + vpcConfig); - + VpcConfig vpcConfig = getVpcConfig(customSecurityGroupId); + Environment environment = getEnvironment(); String kmsKeyArn = brokerKms(tags); - if (functionArn == null && functionCode != null) { buildLogger.addLogEntry("Pushing new Lambda"); @@ -257,8 +260,7 @@ public void brokerLambda() throws IOException { try { buildLogger.addLogEntry("... Resetting execution permissions"); List sidsToRemove = getExistingExecutionPermissionSids(); - - if (sidsToRemove != null) { + if (!sidsToRemove.isEmpty()) { for (String sid: sidsToRemove) { RemovePermissionRequest removeExistingPerms = new RemovePermissionRequest() .withFunctionName(this.configuration.getFunctionName()) @@ -272,16 +274,14 @@ public void brokerLambda() throws IOException { } List permissions = getExecutionPermission(); - - if (permissions != null) { - - + if (!permissions.isEmpty()) { buildLogger.addLogEntry("Adding new execution permission"); for (int i = 0; i < permissions.size(); i++) { LambdaPermission permission = permissions.get(i); if (permission != null) { - String sid = Optional.ofNullable(permission.getSid()).orElse(this.configuration.getFunctionName() + "-InvokePermission-" + i); + String sid = Optional.ofNullable(permission.getSid()) + .orElse(this.configuration.getFunctionName() + "-InvokePermission-" + i); AddPermissionRequest permissionRequest = new AddPermissionRequest() .withFunctionName(this.configuration.getFunctionName()) .withAction(permission.getAction()) @@ -304,20 +304,88 @@ public void brokerLambda() throws IOException { buildLogger.addLogEntry(output.getConfiguration().toString()); } + private Environment getEnvironment() { + HashMap environmentMap = new HashMap<>(); + if (this.configuration.getEnvironment() != null) { + for (KeyValuePair envVar: this.configuration.getEnvironment()) { + environmentMap.put(envVar.getName(), envVar.getValue()); + } + } + return new Environment().withVariables(environmentMap); + } + + private VpcConfig getVpcConfig(String customSecurityGroupId) { + VpcConfig vpcConfig = new VpcConfig(); + if (this.configuration.getSubnetIds() != null) { + vpcConfig.setSubnetIds(this.configuration.getSubnetIds()); + } + if (customSecurityGroupId != null) { + vpcConfig.setSecurityGroupIds(Arrays.asList(customSecurityGroupId)); + } else if (this.configuration.getSecurityGroupIds() != null) { + vpcConfig.setSecurityGroupIds(this.configuration.getSecurityGroupIds()); + } + this.buildLogger.addLogEntry("VPC Configuration: " + vpcConfig); + return vpcConfig; + } + + private String getCustomSecurityGroupId(Map tagMap) { + // Create a custom security group if needed + String customSecurityGroupId = null; + if (this.configuration.getCustomSecurityGroup() != null) { + try { + DescribeSecurityGroupsResult describeSecurityGroupsResult = ec2Client + .describeSecurityGroups(new DescribeSecurityGroupsRequest() + .withFilters( + new Filter("vpc-id").withValues(this.configuration.getCustomSecurityGroup().getVpcId()), + new Filter("group-name").withValues(this.configuration.getFunctionName()))); + if (describeSecurityGroupsResult.getSecurityGroups().size() == 1) { + customSecurityGroupId = describeSecurityGroupsResult.getSecurityGroups().get(0).getGroupId(); + this.buildLogger.addLogEntry("Security group found: " + customSecurityGroupId); + } else { + this.buildLogger.addLogEntry( + "Creating security group with name " + this.configuration.getFunctionName()); + + CreateSecurityGroupResult createSecurityGroupResult = ec2Client.createSecurityGroup( + new CreateSecurityGroupRequest().withGroupName(this.configuration.getFunctionName()) + .withDescription( + "Security group for the " + this.configuration.getFunctionName() + " Lambda function") + .withVpcId(this.configuration.getCustomSecurityGroup().getVpcId())); + + customSecurityGroupId = createSecurityGroupResult.getGroupId(); + this.buildLogger.addLogEntry("... Security group created. ID = " + customSecurityGroupId); + } + } catch (AmazonEC2Exception ex) { + throw new AwsExecException("Error getting security group", ex); + } + + this.buildLogger.addLogEntry("... Updating tags"); + ec2Client.createTags(new CreateTagsRequest() + .withResources(customSecurityGroupId) + .withTags(tagMap.entrySet().stream() + .map(entry -> new com.amazonaws.services.ec2.model.Tag().withKey(entry.getKey()) + .withValue(entry.getValue())) + .collect(Collectors.toList()))); + } + return customSecurityGroupId; + } + private List getExistingExecutionPermissionSids() { - final GetPolicyResult executionPolicyResult = lambdaClient.getPolicy(new GetPolicyRequest().withFunctionName(this.configuration.getFunctionName())); + final GetPolicyResult executionPolicyResult = lambdaClient + .getPolicy(new GetPolicyRequest().withFunctionName(this.configuration.getFunctionName())); if (executionPolicyResult == null) { buildLogger.addLogEntry("Unable to find existing execution policy"); - return null; + return Collections.emptyList(); } try { - final JsonNode executionStatements = this.mapper.readTree(executionPolicyResult.getPolicy()).get("Statement"); - final TypeReference> listRef = new TypeReference>(){}; + final JsonNode executionStatements = this.mapper.readTree(executionPolicyResult.getPolicy()) + .get("Statement"); + final TypeReference> listRef = new TypeReference>() {}; List statements = mapper.readValue(executionStatements.toString(), listRef); return statements.stream().map(it -> it.get("Sid").textValue()).collect(Collectors.toList()); } catch (IOException e) { + LOGGER.debug("Unable to parse existing execution policy", e); buildLogger.addLogEntry("Unable to parse existing execution policy"); - return null; + return Collections.emptyList(); } } @@ -328,26 +396,26 @@ private List getExecutionPermission() { .addLogEntry(String.format("... Using %s for execution permissions", LAMBDA_EXECUTION_PERMISSION)); String template = fileUtil.findFile(LAMBDA_EXECUTION_PERMISSION, false); String mappedPermissionString = this.context.getBambooPropertyHandler().mapInProperties(template); - final TypeReference> listRef = new TypeReference>(){}; + final TypeReference> listRef = new TypeReference>() {}; try { return this.mapper.readValue(mappedPermissionString, listRef); - } - catch (JsonMappingException jsonEx) { - buildLogger.addErrorLogEntry("DEPRECATION WARNING: Singleton permissions are deprecated, please pass execution permissions as an array."); - LambdaPermission singletonPermission = this.mapper.readValue(mappedPermissionString, LambdaPermission.class); + } catch (JsonMappingException jsonEx) { + buildLogger.addErrorLogEntry( + "DEPRECATION WARNING: Singleton permissions are deprecated, please pass execution permissions as an array."); + LambdaPermission singletonPermission = this.mapper + .readValue(mappedPermissionString, LambdaPermission.class); return Collections.singletonList(singletonPermission); - } - catch (Exception ex) { + } catch (Exception ex) { buildLogger.addErrorLogEntry("Error parsing permissions", ex); throw new AwsExecException(ex); } } } catch (IOException ex) { buildLogger.addErrorLogEntry("Error getting execution permissions", ex); - return null; + return Collections.emptyList(); } - return null; + return Collections.emptyList(); } private String brokerKms(List tags) { diff --git a/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaInjectConfiguration.java b/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaInjectConfiguration.java index b1579c0..c20e5af 100644 --- a/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaInjectConfiguration.java +++ b/src/main/java/com/libertymutualgroup/herman/aws/lambda/LambdaInjectConfiguration.java @@ -15,6 +15,7 @@ */ package com.libertymutualgroup.herman.aws.lambda; +import com.amazonaws.services.ec2.model.SecurityGroup; import com.amazonaws.services.ecs.model.KeyValuePair; import com.libertymutualgroup.herman.aws.ecs.broker.iam.IamAppDefinition; import com.libertymutualgroup.herman.aws.ecs.broker.kms.KmsAppDefinition; @@ -29,12 +30,21 @@ public class LambdaInjectConfiguration implements IamAppDefinition, KmsAppDefini private Integer memorySize = 128; private String runtime; private Integer timeout = 5; - private List subnetIds; - private List securityGroupIds; private String iamPolicy; private String assumeRolePolicy; private String useKms = "false"; private String kmsKeyName; + private SecurityGroup customSecurityGroup; + private List subnetIds; + private List securityGroupIds; + + public SecurityGroup getCustomSecurityGroup() { + return customSecurityGroup; + } + + public void setCustomSecurityGroup(SecurityGroup customSecurityGroup) { + this.customSecurityGroup = customSecurityGroup; + } public String getFunctionName() { return functionName; @@ -133,6 +143,14 @@ public void setKmsKeyName(String kmsKeyName) { this.kmsKeyName = kmsKeyName; } + public String getAssumeRolePolicy() { + return assumeRolePolicy; + } + + public void setAssumeRolePolicy(String assumeRolePolicy) { + this.assumeRolePolicy = assumeRolePolicy; + } + public LambdaInjectConfiguration withFunctionName(final String functionName) { this.functionName = functionName; return this; @@ -194,35 +212,33 @@ public LambdaInjectConfiguration withKmsKeyName(final String kmsKeyName) { return this; } - public String getAssumeRolePolicy() { - return assumeRolePolicy; + public LambdaInjectConfiguration withCustomSecurityGroup(final SecurityGroup customSecurityGroup) { + this.customSecurityGroup = customSecurityGroup; + return this; } - public void setAssumeRolePolicy(String assumeRolePolicy) { - this.assumeRolePolicy = assumeRolePolicy; + @Override + public String getAppName() { + return functionName; } @Override public String toString() { return "LambdaInjectConfiguration{" + - "functionName='" + functionName + '\'' + - ", zipFileName='" + zipFileName + '\'' + - ", environment=" + environment + - ", handler='" + handler + '\'' + - ", memorySize=" + memorySize + - ", runtime='" + runtime + '\'' + - ", timeout=" + timeout + - ", subnetIds=" + subnetIds + - ", securityGroupIds=" + securityGroupIds + - ", iamPolicy='" + iamPolicy + '\'' + - ", assumeRolePolicy='" + assumeRolePolicy + '\'' + - ", useKms='" + useKms + '\'' + - ", kmsKeyName='" + kmsKeyName + '\'' + - '}'; - } - - @Override - public String getAppName() { - return functionName; + "functionName='" + functionName + '\'' + + ", zipFileName='" + zipFileName + '\'' + + ", environment=" + environment + + ", handler='" + handler + '\'' + + ", memorySize=" + memorySize + + ", runtime='" + runtime + '\'' + + ", timeout=" + timeout + + ", subnetIds=" + subnetIds + + ", securityGroupIds=" + securityGroupIds + + ", iamPolicy='" + iamPolicy + '\'' + + ", assumeRolePolicy='" + assumeRolePolicy + '\'' + + ", useKms='" + useKms + '\'' + + ", kmsKeyName='" + kmsKeyName + '\'' + + ", customSecurityGroup=" + customSecurityGroup + + '}'; } } diff --git a/src/main/java/com/libertymutualgroup/herman/task/cft/CFTPushTaskProperties.java b/src/main/java/com/libertymutualgroup/herman/task/cft/CFTPushTaskProperties.java index 2063dd5..c401f7b 100644 --- a/src/main/java/com/libertymutualgroup/herman/task/cft/CFTPushTaskProperties.java +++ b/src/main/java/com/libertymutualgroup/herman/task/cft/CFTPushTaskProperties.java @@ -15,8 +15,10 @@ */ package com.libertymutualgroup.herman.task.cft; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.libertymutualgroup.herman.task.common.CommonTaskProperties; +@JsonIgnoreProperties(ignoreUnknown = true) public class CFTPushTaskProperties extends CommonTaskProperties { private String cftPushVariableBrokerLambda; diff --git a/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushPropertyFactory.java b/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushPropertyFactory.java index c302fff..47800cf 100644 --- a/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushPropertyFactory.java +++ b/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushPropertyFactory.java @@ -24,6 +24,10 @@ public class ECSPushPropertyFactory { + private ECSPushPropertyFactory() { + throw new IllegalAccessError("Utility class"); + } + public static ECSPushTaskProperties getTaskProperties(AWSCredentials sessionCredentials, HermanLogger hermanLogger, Regions region) { return getTaskProperties(sessionCredentials, hermanLogger, null, region); } diff --git a/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushTaskProperties.java b/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushTaskProperties.java index 64f0b04..8d1251c 100644 --- a/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushTaskProperties.java +++ b/src/main/java/com/libertymutualgroup/herman/task/ecs/ECSPushTaskProperties.java @@ -15,21 +15,24 @@ */ package com.libertymutualgroup.herman.task.ecs; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.libertymutualgroup.herman.aws.ecs.loadbalancing.SSLCertificate; import com.libertymutualgroup.herman.aws.ecs.logging.SplunkInstance; import com.libertymutualgroup.herman.task.common.CommonTaskProperties; import com.libertymutualgroup.herman.task.newrelic.NewRelicBrokerProperties; - +import java.util.ArrayList; import java.util.List; +@JsonIgnoreProperties(ignoreUnknown = true) public class ECSPushTaskProperties extends CommonTaskProperties { private NewRelicBrokerProperties newRelic; - private List sslCertificates; + private List sslCertificates = new ArrayList<>(); private String ecsConsoleLinkPattern; - private List splunkInstances; + private List splunkInstances = new ArrayList<>(); private String rdsCredentialBrokerImage; private String dnsBrokerLambda; + private List externalElbSecurityGroups = new ArrayList<>(); public NewRelicBrokerProperties getNewRelic() { return newRelic; @@ -80,6 +83,38 @@ public void setDnsBrokerLambda(String dnsBrokerLambda) { this.dnsBrokerLambda = dnsBrokerLambda; } + public List getExternalElbSecurityGroups() { + return externalElbSecurityGroups; + } + + public void setExternalElbSecurityGroups(List externalElbSecurityGroups) { + this.externalElbSecurityGroups = externalElbSecurityGroups; + } + + @Override + public ECSPushTaskProperties withCompany(final String company) { + setCompany(company); + return this; + } + + @Override + public ECSPushTaskProperties withSbu(final String sbu) { + setSbu(sbu); + return this; + } + + @Override + public ECSPushTaskProperties withOrg(final String org) { + setOrg(org); + return this; + } + + @Override + public ECSPushTaskProperties withEngine(final String engine) { + setEngine(engine); + return this; + } + public ECSPushTaskProperties withNewRelic( final NewRelicBrokerProperties newRelic) { this.newRelic = newRelic; @@ -113,6 +148,12 @@ public ECSPushTaskProperties withDnsBrokerLambda(final String dnsBrokerLambda) { return this; } + public ECSPushTaskProperties withExternalElbSecurityGroups( + final List externalElbSecurityGroups) { + this.externalElbSecurityGroups = externalElbSecurityGroups; + return this; + } + @Override public String toString() { return "ECSPushTaskProperties{" + @@ -122,30 +163,7 @@ public String toString() { ", splunkInstances=" + splunkInstances + ", rdsCredentialBrokerImage='" + rdsCredentialBrokerImage + '\'' + ", dnsBrokerLambda='" + dnsBrokerLambda + '\'' + + ", externalElbSecurityGroups=" + externalElbSecurityGroups + "} " + super.toString(); } - - @Override - public ECSPushTaskProperties withCompany(final String company) { - setCompany(company); - return this; - } - - @Override - public ECSPushTaskProperties withSbu(final String sbu) { - setSbu(sbu); - return this; - } - - @Override - public ECSPushTaskProperties withOrg(final String org) { - setOrg(org); - return this; - } - - @Override - public ECSPushTaskProperties withEngine(final String engine) { - setEngine(engine); - return this; - } } \ No newline at end of file diff --git a/src/main/java/com/libertymutualgroup/herman/task/s3/S3CreateTask.java b/src/main/java/com/libertymutualgroup/herman/task/s3/S3CreateTask.java index 0ea8531..2327efa 100644 --- a/src/main/java/com/libertymutualgroup/herman/task/s3/S3CreateTask.java +++ b/src/main/java/com/libertymutualgroup/herman/task/s3/S3CreateTask.java @@ -77,7 +77,8 @@ CommonTaskProperties getTaskProperties(AWSCredentials sessionCredentials, Herman ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); return objectMapper.readValue(s3CreateTaskPropertiesYml, CommonTaskProperties.class); } catch (Exception ex) { - throw new RuntimeException("Error getting S3 Create Task Task Properties", ex); + hermanLogger.addErrorLogEntry("Error getting S3 Create Task Task Properties. Continuing...", ex); + return null; } } } diff --git a/src/main/java/com/libertymutualgroup/herman/util/ConfigurationUtil.java b/src/main/java/com/libertymutualgroup/herman/util/ConfigurationUtil.java index 443c551..5ee3451 100644 --- a/src/main/java/com/libertymutualgroup/herman/util/ConfigurationUtil.java +++ b/src/main/java/com/libertymutualgroup/herman/util/ConfigurationUtil.java @@ -22,6 +22,10 @@ public class ConfigurationUtil { + private ConfigurationUtil() { + throw new IllegalAccessError("Utility class"); + } + private static final String CONFIG_FILE = "properties.yml"; private static final String ECR_POLICY_FILE = "ecr-policy.json"; private static final String KMS_POLICY_FILE = "kms-policy.json"; @@ -33,7 +37,7 @@ public static String getHermanConfigurationAsString(AWSCredentials sessionCreden public static String getHermanConfigurationAsString(AWSCredentials sessionCredentials, HermanLogger hermanLogger, String customConfigurationBucket, Regions region) { try { - String hermanConfigBucket = getConfigurationBucketName(sessionCredentials, customConfigurationBucket); + String hermanConfigBucket = getConfigurationBucketName(sessionCredentials, customConfigurationBucket, region); hermanLogger.addLogEntry(String.format("... Using task config from S3 bucket %s: %s", hermanConfigBucket, CONFIG_FILE)); AmazonS3 s3Client = AmazonS3ClientBuilder.standard() @@ -49,7 +53,7 @@ public static String getHermanConfigurationAsString(AWSCredentials sessionCreden public static String getECRPolicyAsString(AWSCredentials sessionCredentials, HermanLogger hermanLogger, String customConfigurationBucket, Regions region) { try { - String configBucket = getConfigurationBucketName(sessionCredentials, customConfigurationBucket); + String configBucket = getConfigurationBucketName(sessionCredentials, customConfigurationBucket, region); hermanLogger.addLogEntry(String.format("... Using ECR policy file from S3 bucket %s: %s", configBucket, ECR_POLICY_FILE)); AmazonS3 s3Client = AmazonS3ClientBuilder.standard() @@ -65,7 +69,7 @@ public static String getECRPolicyAsString(AWSCredentials sessionCredentials, Her public static String getKMSPolicyAsString(AWSCredentials sessionCredentials, HermanLogger hermanLogger, String customConfigurationBucket, Regions region) { try { - String configBucket = getConfigurationBucketName(sessionCredentials, customConfigurationBucket); + String configBucket = getConfigurationBucketName(sessionCredentials, customConfigurationBucket, region); hermanLogger.addLogEntry(String.format("... Using KMS policy file from S3 bucket %s: %s", configBucket, KMS_POLICY_FILE)); AmazonS3 s3Client = AmazonS3ClientBuilder.standard() @@ -79,8 +83,7 @@ public static String getKMSPolicyAsString(AWSCredentials sessionCredentials, Her } } - - private static String getConfigurationBucketName(AWSCredentials sessionCredentials, String customConfigurationBucket) + private static String getConfigurationBucketName(AWSCredentials sessionCredentials, String customConfigurationBucket, Regions region) throws IOException { AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(sessionCredentials)) @@ -96,7 +99,10 @@ private static String getConfigurationBucketName(AWSCredentials sessionCredentia if (customConfigurationBucket != null) { hermanConfigBucket = customConfigurationBucket; } else { - hermanConfigBucket = String.format("herman-configuration-%s-%s", account, versionProperties.getProperty("version").toLowerCase()); + hermanConfigBucket = String.format("herman-configuration-%s-%s-%s", + account, + region.getName(), + versionProperties.getProperty("version").toLowerCase()); } return hermanConfigBucket; } diff --git a/src/main/java/com/libertymutualgroup/herman/util/HttpStatusUtil.java b/src/main/java/com/libertymutualgroup/herman/util/HttpStatusUtil.java index ffcce5f..13f6f3f 100644 --- a/src/main/java/com/libertymutualgroup/herman/util/HttpStatusUtil.java +++ b/src/main/java/com/libertymutualgroup/herman/util/HttpStatusUtil.java @@ -16,6 +16,11 @@ package com.libertymutualgroup.herman.util; public class HttpStatusUtil { + + private HttpStatusUtil() { + throw new IllegalAccessError("Utility class"); + } + public static boolean isSuccessful(int statusCode) { return statusCode >= 200 && statusCode < 300; } diff --git a/src/main/resources/config/samples/properties.yml b/src/main/resources/config/samples/properties.yml index 5e8d781..fda54bd 100644 --- a/src/main/resources/config/samples/properties.yml +++ b/src/main/resources/config/samples/properties.yml @@ -6,8 +6,6 @@ ecsConsoleLinkPattern: # URL pattern to access the AWS console rdsCredentialBrokerImage: # RDS Cred Broker image location cftPushVariableBrokerLambda: # CFT Push Variable Broker Lambda name dnsBrokerLambda: # DNS Broker Lambda name -awsAccount: - prod: # Prod AWS account newRelic: accountId: # New Relic Account ID nrLambda: # New Relic Broker Lambda name diff --git a/src/test/java/com/libertymutualgroup/herman/aws/ecs/EcsClusterIntrospectorTest.java b/src/test/java/com/libertymutualgroup/herman/aws/ecs/EcsClusterIntrospectorTest.java index 40a1cb7..116732a 100644 --- a/src/test/java/com/libertymutualgroup/herman/aws/ecs/EcsClusterIntrospectorTest.java +++ b/src/test/java/com/libertymutualgroup/herman/aws/ecs/EcsClusterIntrospectorTest.java @@ -1,5 +1,8 @@ package com.libertymutualgroup.herman.aws.ecs; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + import com.amazonaws.services.cloudformation.AmazonCloudFormation; import com.amazonaws.services.cloudformation.model.DescribeStackResourcesRequest; import com.amazonaws.services.cloudformation.model.DescribeStackResourcesResult; @@ -10,28 +13,20 @@ import com.amazonaws.services.cloudformation.model.StackResource; import com.amazonaws.services.cloudformation.model.Tag; import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest; -import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult; import com.amazonaws.services.ec2.model.DescribeSubnetsResult; import com.amazonaws.services.ec2.model.DescribeVpcsResult; -import com.amazonaws.services.ec2.model.Filter; -import com.amazonaws.services.ec2.model.SecurityGroup; import com.amazonaws.services.ec2.model.Subnet; import com.amazonaws.services.ec2.model.Vpc; import com.libertymutualgroup.herman.aws.ecs.cluster.EcsClusterIntrospector; import com.libertymutualgroup.herman.aws.ecs.cluster.EcsClusterMetadata; import com.libertymutualgroup.herman.logging.HermanLogger; import com.libertymutualgroup.herman.logging.SysoutLogger; +import java.util.List; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.when; - public class EcsClusterIntrospectorTest { HermanLogger logger = new SysoutLogger(); @@ -90,14 +85,6 @@ public void testInspect() { Subnet subnet = new Subnet().withTags(subnetTag).withVpcId("vpc12345"); when(ec2Client.describeSubnets()).thenReturn(new DescribeSubnetsResult().withSubnets(subnet)); - Filter filter1 = new Filter().withName("tag:Name").withValues("aws-shared-external-elb-nonprod-1"); - when(ec2Client.describeSecurityGroups(new DescribeSecurityGroupsRequest().withFilters(filter1))).thenReturn( - new DescribeSecurityGroupsResult().withSecurityGroups(new SecurityGroup().withGroupId("sg99998"))); - - Filter filter2 = new Filter().withName("tag:Name").withValues("aws-shared-external-elb-nonprod-2"); - when(ec2Client.describeSecurityGroups(new DescribeSecurityGroupsRequest().withFilters(filter2))).thenReturn( - new DescribeSecurityGroupsResult().withSecurityGroups(new SecurityGroup().withGroupId("sg99999"))); - //WHEN EcsClusterMetadata meta = introspector.introspect(stackName); @@ -112,7 +99,6 @@ public void testInspect() { assertEquals("LMB", meta.getNewrelicSbuTag()); assertEquals("sg34567", meta.getRdsSecurityGroup()); assertEquals("vpc12345", meta.getVpcId()); - assertEquals(2, meta.getAkamaiSecurityGroup().size()); assertEquals(1, meta.getElbSecurityGroups().size()); assertEquals(1, meta.getElbSubnets().size()); } diff --git a/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandlerTest.java b/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandlerTest.java index 6e5e4a1..9881f45 100644 --- a/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandlerTest.java +++ b/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerHandlerTest.java @@ -1,6 +1,10 @@ package com.libertymutualgroup.herman.aws.ecs.loadbalancing; -import com.amazonaws.services.cloudformation.AmazonCloudFormation; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.amazonaws.services.elasticloadbalancing.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancing.model.AddTagsRequest; import com.amazonaws.services.elasticloadbalancing.model.ApplySecurityGroupsToLoadBalancerRequest; @@ -14,7 +18,6 @@ import com.amazonaws.services.elasticloadbalancing.model.Listener; import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerDescription; import com.amazonaws.services.elasticloadbalancing.model.Tag; -import com.amazonaws.services.identitymanagement.AmazonIdentityManagement; import com.libertymutualgroup.herman.aws.ecs.EcsDefinitionParser; import com.libertymutualgroup.herman.aws.ecs.EcsPushDefinition; import com.libertymutualgroup.herman.aws.ecs.PropertyHandler; @@ -22,23 +25,17 @@ import com.libertymutualgroup.herman.aws.ecs.cluster.EcsClusterMetadata; import com.libertymutualgroup.herman.logging.HermanLogger; import com.libertymutualgroup.herman.logging.SysoutLogger; -import com.libertymutualgroup.herman.task.common.CommonTaskProperties; -import org.apache.commons.io.FileUtils; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - +import com.libertymutualgroup.herman.task.ecs.ECSPushTaskProperties; import java.io.File; import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.List; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; public class EcsLoadBalancerHandlerTest { @@ -47,10 +44,6 @@ public class EcsLoadBalancerHandlerTest { @Mock AmazonElasticLoadBalancing elbClient; @Mock - AmazonIdentityManagement iamClient; - @Mock - AmazonCloudFormation cftClient; - @Mock CertHandler certHandler; @Mock DnsRegistrar dnsRegistrar; @@ -61,14 +54,14 @@ public class EcsLoadBalancerHandlerTest { public void setup() { MockitoAnnotations.initMocks(this); - CommonTaskProperties taskProperties = new CommonTaskProperties() + ECSPushTaskProperties taskProperties = new ECSPushTaskProperties() .withCompany("lm") .withOrg("LMB") .withSbu("CI"); handler = new EcsLoadBalancerHandler(elbClient, certHandler, dnsRegistrar, logger, taskProperties); when(certHandler.deriveCert("HTTPS", "np-lmb.lmig.com", "my-app-dev")) - .thenReturn(new DeriveCertResult().withCertArn("arn:somecert")); + .thenReturn(new SSLCertificate().withArn("arn:somecert")); } @Test @@ -77,7 +70,10 @@ public void elbCreate() throws IOException { EcsClusterMetadata meta = generateMetadata(); EcsPushDefinition pushDef = loadTemplate("template.yml"); - LoadBalancerDescription balancerDescription = new LoadBalancerDescription().withDNSName("https://some.aws.url"); + LoadBalancerDescription balancerDescription = new LoadBalancerDescription() + .withDNSName("https://some.aws.url") + .withCanonicalHostedZoneNameID("123") + .withScheme("internal"); DescribeLoadBalancersResult elbResult = new DescribeLoadBalancersResult() .withLoadBalancerDescriptions(balancerDescription); when(elbClient @@ -90,8 +86,10 @@ public void elbCreate() throws IOException { // THEN // DNS Registered verify(dnsRegistrar).registerDns( - pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix(), - "https://some.aws.url", pushDef.getService().getUrlPrefixOverride(), meta.getClusterCftStackTags()); + pushDef.getAppName(), + "classic", + "HTTPS", + pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix()); // Create ELB List elbTags = new ArrayList<>(); elbTags.add(new Tag().withKey("lm_cluster").withValue("some-cluster")); @@ -116,7 +114,10 @@ public void elbUpdate() throws IOException { EcsPushDefinition pushDef = loadTemplate("template.yml"); when(elbClient.createLoadBalancer(any())).thenThrow(new DuplicateLoadBalancerNameException("exists")); - LoadBalancerDescription balancerDescription = new LoadBalancerDescription().withDNSName("https://some.aws.url"); + LoadBalancerDescription balancerDescription = new LoadBalancerDescription() + .withDNSName("https://some.aws.url") + .withCanonicalHostedZoneNameID("123") + .withScheme("internal"); DescribeLoadBalancersResult elbResult = new DescribeLoadBalancersResult() .withLoadBalancerDescriptions(balancerDescription); when(elbClient @@ -129,8 +130,10 @@ public void elbUpdate() throws IOException { // THEN // DNS Registered verify(dnsRegistrar).registerDns( - pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix(), - "https://some.aws.url", pushDef.getService().getUrlPrefixOverride(), meta.getClusterCftStackTags()); + pushDef.getAppName(), + "classic", + "HTTPS", + pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix()); // Update ELB verify(elbClient).deleteLoadBalancerListeners(any()); diff --git a/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2HandlerTest.java b/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2HandlerTest.java index 8ffb5ee..775bc32 100644 --- a/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2HandlerTest.java +++ b/src/test/java/com/libertymutualgroup/herman/aws/ecs/loadbalancing/EcsLoadBalancerV2HandlerTest.java @@ -1,6 +1,10 @@ package com.libertymutualgroup.herman.aws.ecs.loadbalancing; -import com.amazonaws.services.cloudformation.AmazonCloudFormation; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.model.Action; import com.amazonaws.services.elasticloadbalancingv2.model.ActionTypeEnum; @@ -28,7 +32,6 @@ import com.amazonaws.services.elasticloadbalancingv2.model.SetSubnetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.Tag; import com.amazonaws.services.elasticloadbalancingv2.model.TargetGroup; -import com.amazonaws.services.identitymanagement.AmazonIdentityManagement; import com.libertymutualgroup.herman.aws.ecs.EcsDefinitionParser; import com.libertymutualgroup.herman.aws.ecs.EcsPushDefinition; import com.libertymutualgroup.herman.aws.ecs.PropertyHandler; @@ -36,23 +39,17 @@ import com.libertymutualgroup.herman.aws.ecs.cluster.EcsClusterMetadata; import com.libertymutualgroup.herman.logging.HermanLogger; import com.libertymutualgroup.herman.logging.SysoutLogger; -import com.libertymutualgroup.herman.task.common.CommonTaskProperties; -import org.apache.commons.io.FileUtils; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - +import com.libertymutualgroup.herman.task.ecs.ECSPushTaskProperties; import java.io.File; import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.List; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; public class EcsLoadBalancerV2HandlerTest { @@ -61,10 +58,6 @@ public class EcsLoadBalancerV2HandlerTest { @Mock AmazonElasticLoadBalancing elbClient; @Mock - AmazonIdentityManagement iamClient; - @Mock - AmazonCloudFormation cftClient; - @Mock CertHandler certHandler; @Mock DnsRegistrar dnsRegistrar; @@ -75,7 +68,7 @@ public class EcsLoadBalancerV2HandlerTest { public void setup() { MockitoAnnotations.initMocks(this); - CommonTaskProperties taskProperties = new CommonTaskProperties() + ECSPushTaskProperties taskProperties = new ECSPushTaskProperties() .withCompany("lm") .withOrg("LMB") .withSbu("CI"); @@ -83,7 +76,7 @@ public void setup() { handler = new EcsLoadBalancerV2Handler(elbClient, certHandler, dnsRegistrar, logger, taskProperties); when(certHandler.deriveCert("HTTPS", "np-lmb.lmig.com", "my-app-dev")) - .thenReturn(new DeriveCertResult().withCertArn("arn:somecert")); + .thenReturn(new SSLCertificate().withArn("arn:somecert")); } @@ -94,7 +87,9 @@ public void executeElbUpdate() throws IOException { EcsPushDefinition pushDef = loadTemplate("template.yml"); LoadBalancer balancer = new LoadBalancer().withDNSName("https://some.aws.url") - .withLoadBalancerArn("arn:balancer"); + .withLoadBalancerArn("arn:balancer") + .withCanonicalHostedZoneId("123") + .withScheme("internal"); DescribeLoadBalancersResult balancersResult = new DescribeLoadBalancersResult().withLoadBalancers(balancer); when(elbClient.describeLoadBalancers(any(DescribeLoadBalancersRequest.class))).thenReturn(balancersResult); @@ -111,10 +106,11 @@ public void executeElbUpdate() throws IOException { //THEN // DNS Registered - verify(dnsRegistrar) - .registerDns(pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix(), - "https://some.aws.url", - pushDef.getService().getUrlPrefixOverride(), meta.getClusterCftStackTags()); + verify(dnsRegistrar).registerDns( + pushDef.getAppName(), + "application", + "HTTPS", + pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix()); // Update Target Group ModifyTargetGroupRequest modifyTargetGroupRequest = new ModifyTargetGroupRequest() @@ -167,7 +163,9 @@ public void executeElbCreate() throws IOException { when(elbClient.createTargetGroup(any(CreateTargetGroupRequest.class))).thenReturn(createTargetGroupResult); LoadBalancer balancer = new LoadBalancer().withDNSName("https://some.aws.url") - .withLoadBalancerArn("arn:balancer"); + .withLoadBalancerArn("arn:balancer") + .withCanonicalHostedZoneId("123") + .withScheme("internal"); CreateLoadBalancerResult balancerResult = new CreateLoadBalancerResult().withLoadBalancers(balancer); when(elbClient.createLoadBalancer(any(CreateLoadBalancerRequest.class))).thenReturn(balancerResult); @@ -182,10 +180,11 @@ public void executeElbCreate() throws IOException { //THEN // DNS Registered - verify(dnsRegistrar) - .registerDns(pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix(), - "https://some.aws.url", - pushDef.getService().getUrlPrefixOverride(), meta.getClusterCftStackTags()); + verify(dnsRegistrar).registerDns( + pushDef.getAppName(), + "application", + "HTTPS", + pushDef.getService().getUrlPrefixOverride() + "." + pushDef.getService().getUrlSuffix()); CreateTargetGroupRequest ctgr = new CreateTargetGroupRequest().withName(pushDef.getAppName()) .withHealthCheckIntervalSeconds(pushDef.getService().getHealthCheck().getInterval())