Load hdfs config files and set key-value pairs in SmartConf (#1593)

Intel-bigdata · Feb 9, 2018 · 8ab5215 · 8ab5215
1 parent f575d47
commit 8ab5215
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 10 deletions.
diff --git a/smart-agent/src/main/java/org/smartdata/agent/SmartAgent.java b/smart-agent/src/main/java/org/smartdata/agent/SmartAgent.java
@@ -72,6 +72,7 @@ public static void main(String[] args) throws IOException {
     SmartAgent agent = new SmartAgent();
 
     SmartConf conf = (SmartConf) new GenericOptionsParser(new SmartConf(), args).getConfiguration();
+
     String[] masters = AgentUtils.getMasterAddress(conf);
     if (masters == null) {
       throw new IOException("No master address found!");
@@ -82,27 +83,29 @@ public static void main(String[] args) throws IOException {
     String agentAddress = AgentUtils.getAgentAddress(conf);
     LOG.info("Agent address: " + agentAddress);
 
+    HadoopUtil.setSmartConfByHadoop(conf);
     agent.authentication(conf);
 
     agent.start(AgentUtils.overrideRemoteAddress(ConfigFactory.load(AgentConstants.AKKA_CONF_FILE),
         agentAddress), AgentUtils.getMasterActorPaths(masters), conf);
   }
 
+  //TODO: remove loadHadoopConf
   private void authentication(SmartConf conf) throws IOException {
     if (!SecurityUtil.isSecurityEnabled(conf)) {
       return;
     }
 
     // Load Hadoop configuration files
-    String hadoopConfPath = conf.get(SmartConfKeys.SMART_HADOOP_CONF_DIR_KEY);
     try {
-      HadoopUtil.loadHadoopConf(hadoopConfPath, conf);
+      HadoopUtil.loadHadoopConf(conf);
     } catch (IOException e) {
       LOG.info("Running in secure mode, but cannot find Hadoop configuration file. "
-          + "Please config smart.hadoop.conf.path property in smart-site.xml.");
+              + "Please config smart.hadoop.conf.path property in smart-site.xml.");
       conf.set("hadoop.security.authentication", "kerberos");
       conf.set("hadoop.security.authorization", "true");
     }
+
     UserGroupInformation.setConfiguration(conf);
 
     String keytabFilename = conf.get(SmartConfKeys.SMART_AGENT_KEYTAB_FILE_KEY);

diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HadoopUtil.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HadoopUtil.java
@@ -102,6 +102,37 @@ public static void loadHadoopConf(String hadoopConfPath, Configuration conf)
     }
   }
 
+  /**
+   * Load hadoop configure files from path configured in conf and override same key's value.
+   *
+   * @param conf
+   */
+  public static void loadHadoopConf(Configuration conf) throws IOException {
+    String hadoopConfPath = conf.get(SmartConfKeys.SMART_HADOOP_CONF_DIR_KEY);
+    HdfsConfiguration hadoopConf = getHadoopConf(hadoopConfPath);
+    if (hadoopConf != null) {
+      for (Map.Entry<String, String> entry : hadoopConf) {
+        String key = entry.getKey();
+        conf.set(key, entry.getValue());
+      }
+    }
+  }
+
+  public static void setSmartConfByHadoop(SmartConf conf) {
+    try {
+      if (conf.get(SmartConfKeys.SMART_HADOOP_CONF_DIR_KEY) != null) {
+        HadoopUtil.loadHadoopConf(conf);
+        URI nnUri = HadoopUtil.getNameNodeUri(conf);
+        if (nnUri != null) {
+          conf.set(SmartConfKeys.SMART_DFS_NAMENODE_RPCSERVER_KEY,
+                  nnUri.toString());
+        }
+      }
+    } catch (IOException ex) {
+      LOG.error("Load hadoop conf in {} error", conf.get(SmartConfKeys.SMART_HADOOP_CONF_DIR_KEY));
+    }
+  }
+
   /**
    * Get hadoop configuration from the configure files in the given directory.
    *

diff --git a/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HdfsStatesUpdateService.java b/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HdfsStatesUpdateService.java
@@ -73,6 +73,7 @@ public HdfsStatesUpdateService(SmartContext context, MetaStore metaStore) {
    *
    * @return true if initialized successfully
    */
+  //@TODO: remove loadHadoopConf because it is done in Smart Server
   @Override
   public void init() throws IOException {
     LOG.info("Initializing ...");

diff --git a/smart-server/src/main/java/org/smartdata/server/SmartDaemon.java b/smart-server/src/main/java/org/smartdata/server/SmartDaemon.java
@@ -20,6 +20,7 @@
 import com.hazelcast.core.HazelcastInstance;
 import org.smartdata.SmartContext;
 import org.smartdata.conf.SmartConf;
+import org.smartdata.hdfs.HadoopUtil;
 import org.smartdata.server.cluster.ClusterMembershipListener;
 import org.smartdata.server.cluster.HazelcastInstanceProvider;
 import org.smartdata.server.cluster.HazelcastWorker;
@@ -43,7 +44,11 @@ public void start() throws IOException, InterruptedException {
       SmartServer.main(args);
     } else {
       instance.getCluster().addMembershipListener(new ClusterMembershipListener(this));
-      this.hazelcastWorker = new HazelcastWorker(new SmartContext(new SmartConf()));
+
+      SmartConf conf = new SmartConf();
+      HadoopUtil.setSmartConfByHadoop(conf);
+
+      this.hazelcastWorker = new HazelcastWorker(new SmartContext(conf));
       this.hazelcastWorker.start();
     }
   }

diff --git a/smart-server/src/main/java/org/smartdata/server/SmartServer.java b/smart-server/src/main/java/org/smartdata/server/SmartServer.java
@@ -86,6 +86,7 @@ public SmartServer(SmartConf conf) {
   public void initWith() throws Exception {
     LOG.info("Start Init Smart Server");
 
+    HadoopUtil.setSmartConfByHadoop(conf);
     authentication();
     MetaStore metaStore = MetaStoreUtils.getDBAdapter(conf);
     context = new ServerContext(conf, metaStore);
@@ -270,15 +271,15 @@ private void authentication() throws IOException {
     if (!SecurityUtil.isSecurityEnabled(conf)) {
       return;
     }
+
     // Load Hadoop configuration files
-    String hadoopConfPath = conf.get(SmartConfKeys.SMART_HADOOP_CONF_DIR_KEY);
     try {
-      HadoopUtil.loadHadoopConf(hadoopConfPath, conf);
+      HadoopUtil.loadHadoopConf(conf);
     } catch (IOException e) {
-        LOG.info("Running in secure mode, but cannot find Hadoop configuration file. "
-            + "Please config smart.hadoop.conf.path property in smart-site.xml.");
-        conf.set("hadoop.security.authentication", "kerberos");
-        conf.set("hadoop.security.authorization", "true");
+      LOG.info("Running in secure mode, but cannot find Hadoop configuration file. "
+              + "Please config smart.hadoop.conf.path property in smart-site.xml.");
+      conf.set("hadoop.security.authentication", "kerberos");
+      conf.set("hadoop.security.authorization", "true");
     }
 
     UserGroupInformation.setConfiguration(conf);