From d96bd09921bf4d81862d1390b00c5850d925a9fa Mon Sep 17 00:00:00 2001 From: avinashk07 Date: Mon, 13 Apr 2020 18:50:02 -0700 Subject: [PATCH 1/2] ref #FLUME-3361 Added support for environment variable in ZooKeeper based conf by setting -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties . This is consistent with the way environment variables are supported in local file based Flume agent conf. --- ...bstractZooKeeperConfigurationProvider.java | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java b/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java index f193f9f90a..db9239ed1f 100644 --- a/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java +++ b/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java @@ -33,6 +33,9 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * ZooKeeper based configuration implementation provider. * @@ -70,6 +73,9 @@ public abstract class AbstractZooKeeperConfigurationProvider extends protected final String zkConnString; + private static final String DEFAULT_PROPERTIES_IMPLEMENTATION = "java.util.Properties"; + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractZooKeeperConfigurationProvider.class); + protected AbstractZooKeeperConfigurationProvider(String agentName, String zkConnString, String basePath) { super(agentName); @@ -90,14 +96,24 @@ protected CuratorFramework createClient() { protected FlumeConfiguration configFromBytes(byte[] configData) throws IOException { - Map configMap; - if (configData == null || configData.length == 0) { - configMap = Collections.emptyMap(); - } else { - String fileContent = new String(configData, Charsets.UTF_8); - Properties properties = new Properties(); - properties.load(new StringReader(fileContent)); - configMap = toMap(properties); + Map configMap = Collections.emptyMap(); + if (configData != null && configData.length > 0) { + try{ + String fileContent = new String(configData, Charsets.UTF_8); + String resolverClassName = System.getProperty("propertiesImplementation", + DEFAULT_PROPERTIES_IMPLEMENTATION); + Class propsclass = Class.forName(resolverClassName) + .asSubclass(Properties.class); + Properties properties = propsclass.newInstance(); + properties.load(new StringReader(fileContent)); + configMap = toMap(properties); + } catch(ClassNotFoundException e){ + LOGGER.error("Configuartion resolver class not found", e); + } catch(InstantiationException e){ + LOGGER.error("Instantiation exception", e); + } catch(IllegalAccessException e){ + LOGGER.error("Illegal access exception", e); + } } return new FlumeConfiguration(configMap); } From 2a987ae316dc2a71f56986040e339e06638fc50a Mon Sep 17 00:00:00 2001 From: avinashk07 Date: Mon, 13 Apr 2020 20:39:50 -0700 Subject: [PATCH 2/2] #FLUME-3361 Added support for environment variable in ZooKeeper based conf by setting -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties . This is consistent with the way environment variables are supported in local file based Flume agent conf. Also fixed code style issues. --- .../node/AbstractZooKeeperConfigurationProvider.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java b/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java index db9239ed1f..5244fd4af2 100644 --- a/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java +++ b/flume-ng-node/src/main/java/org/apache/flume/node/AbstractZooKeeperConfigurationProvider.java @@ -74,7 +74,8 @@ public abstract class AbstractZooKeeperConfigurationProvider extends protected final String zkConnString; private static final String DEFAULT_PROPERTIES_IMPLEMENTATION = "java.util.Properties"; - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractZooKeeperConfigurationProvider.class); + private static final Logger LOGGER = LoggerFactory.getLogger( + AbstractZooKeeperConfigurationProvider.class); protected AbstractZooKeeperConfigurationProvider(String agentName, String zkConnString, String basePath) { @@ -98,7 +99,7 @@ protected FlumeConfiguration configFromBytes(byte[] configData) throws IOException { Map configMap = Collections.emptyMap(); if (configData != null && configData.length > 0) { - try{ + try { String fileContent = new String(configData, Charsets.UTF_8); String resolverClassName = System.getProperty("propertiesImplementation", DEFAULT_PROPERTIES_IMPLEMENTATION); @@ -107,11 +108,11 @@ protected FlumeConfiguration configFromBytes(byte[] configData) Properties properties = propsclass.newInstance(); properties.load(new StringReader(fileContent)); configMap = toMap(properties); - } catch(ClassNotFoundException e){ + } catch (ClassNotFoundException e) { LOGGER.error("Configuartion resolver class not found", e); - } catch(InstantiationException e){ + } catch (InstantiationException e) { LOGGER.error("Instantiation exception", e); - } catch(IllegalAccessException e){ + } catch (IllegalAccessException e) { LOGGER.error("Illegal access exception", e); } }