fix filedescriptor leaks in lucene metadata store and filesystem mode…

…l store
driftserver · Jun 6, 2019 · a39fe0e · a39fe0e
1 parent 509d98c
commit a39fe0e
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 29 deletions.
diff --git a/drift-core/drift-core-domain/src/main/java/io/drift/core/store/ModelStore.java b/drift-core/drift-core-domain/src/main/java/io/drift/core/store/ModelStore.java
@@ -85,4 +85,7 @@ public ModelStore withFormatForClass(Class<?> _class, String format) {
 		return this;
 	}
 
+	public void shutDown() {
+		metaDataStorage.shutDown();
+	}
 }
diff --git a/...e/drift-core-domain/src/main/java/io/drift/core/store/storage/FileSystemModelStorage.java b/...e/drift-core-domain/src/main/java/io/drift/core/store/storage/FileSystemModelStorage.java
@@ -25,7 +25,7 @@ private Path getPath(StoragePath storagePath, StorageId storageId, String format
 	@Override
 	public String readContent(StorageId storageId, StoragePath storagePath, String format) throws ModelStorageException {
 		try {
-			return Files.lines(getPath(storagePath, storageId, format)).collect(Collectors.joining(System.lineSeparator()));
+			return Files.readAllLines(getPath(storagePath, storageId, format)).stream().collect(Collectors.joining(System.lineSeparator()));
 		} catch (IOException e) {
 			throw new ModelStorageException(e);
 		}

diff --git a/drift-core/drift-core-domain/src/test/java/io/drift/core/store/StoreTest.java b/drift-core/drift-core-domain/src/test/java/io/drift/core/store/StoreTest.java
@@ -1,35 +1,82 @@
 package io.drift.core.store;
 
 import io.drift.core.store.serialization.JsonModelSerializer;
+import io.drift.core.store.serialization.YamlModelSerializer;
 import io.drift.core.store.storage.FileSystemModelStorage;
+import io.drift.core.store.storage.LuceneMetaDataStorage;
 import io.drift.core.store.storage.StorageId;
 import io.drift.core.store.storage.StoragePath;
+import io.drift.core.system.SystemDescription;
 import junit.framework.TestCase;
 import org.junit.Assert;
 
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static io.drift.core.store.serialization.JsonModelSerializer.JSON_FORMAT;
+import static io.drift.core.store.serialization.YamlModelSerializer.YAML_FORMAT;
 
 public class StoreTest extends TestCase {
 
-	private Path modelStorageDir;
+	private Path storeBaseDir;
+	private Path luceneIndexDir;
+	private Path fileStorageDir;
+
+	private ModelStore store;
 
 	protected void setUp() throws Exception {
-		modelStorageDir = Files.createTempDirectory("model");
-		System.out.println("modelStorageDir: " + modelStorageDir);
+		storeBaseDir = Files.createTempDirectory("store");
+
+		luceneIndexDir = storeBaseDir.resolve("lucene");
+		fileStorageDir = storeBaseDir.resolve("files");
+
+		Files.createDirectories(luceneIndexDir);
+		Files.createDirectories(fileStorageDir);
+
+		System.out.println("storageBaseDir: " + storeBaseDir);
+
+		store = createStore();
+	}
+
+	protected void tearDown() throws Exception {
+
+		store.shutDown();
+
+		// TimeUnit.SECONDS.sleep(5);
+
+
+		System.out.println("deleting: " + storeBaseDir);
+		// Files.delete(storeBaseDir);
+		recursiveDelete(storeBaseDir);
+		return;
 	}
 
 	private ModelStore createStore() {
-		ModelStore store = new ModelStore();
 
-		store.getSerializationManager().registerSerializer(new JsonModelSerializer());
-		store.getModelStorageManager().registerStorage(new FileSystemModelStorage(modelStorageDir));
-		return store;
+		JsonModelSerializer jsonModelSerializer = new JsonModelSerializer();
+		YamlModelSerializer yamlModelSerializer = new YamlModelSerializer();
+
+		return new ModelStore()
+
+				.withSerializer(jsonModelSerializer)
+				.withSerializer(yamlModelSerializer)
+
+				.withDefaultFormat(JSON_FORMAT)
+				.withFormatForClass(SystemDescription.class, YAML_FORMAT)
+
+				.withModelStorage(new FileSystemModelStorage(fileStorageDir))
+
+				.withMetaDataStorage(new LuceneMetaDataStorage(luceneIndexDir));
+
 	}
 
 	public void testSaveThenGetMetaData() throws ModelStoreException {
-		ModelStore store = createStore();
 
 		StorageId storageId = new StorageId(UUID.randomUUID().toString());
 		StoragePath parentPath = new StoragePath(new StorageId("a"), new StorageId("b"));
@@ -49,7 +96,6 @@ public void testSaveThenGetMetaData() throws ModelStoreException {
 	}
 
 	public void testSaveThenLoad() throws ModelStoreException {
-		ModelStore store = createStore();
 
 		StorageId storageId = new StorageId("id");
 		StoragePath parentPath = new StoragePath(new StorageId("a"), new StorageId("b"));
@@ -72,4 +118,22 @@ public void testSaveThenLoad() throws ModelStoreException {
 
 	}
 
+	private void recursiveDelete(Path baseDir) throws IOException {
+		Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
+
+			@Override
+			public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+				Files.delete(file);
+				return FileVisitResult.CONTINUE;
+			}
+
+			@Override
+			public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+				Files.delete(dir);
+				// System.out.println("deleting dir " + dir);
+				return FileVisitResult.CONTINUE;
+			}
+		});
+	}
+
 }
diff --git a/drift-jdbc/drift-jdbc-domain/src/test/java/io/drift/plugin/jdbc/StoreTest.java b/drift-jdbc/drift-jdbc-domain/src/test/java/io/drift/plugin/jdbc/StoreTest.java
@@ -95,23 +95,7 @@ public void test_parse_system_description() throws ModelStoreException {
         YamlModelSerializer yamlModelSerializer = createYamlSerializer();
 
         String content = "---\n" +
-                "\n" +
-                "environments:\n" +
-                "- key: LOCAL\n" +
-                "  name: Local\n" +
-                "- key: DEV\n" +
-                "  name: Develop\n" +
-                "\n" +
-                "subSystems:\n" +
-                "- key: PETSDB\n" +
-                "  type: jdbc\n" +
-                "  name: Pets Database\n" +
-                "- key: PRODUCTSDB\n" +
-                "  type: jdbc\n" +
-                "  name: Product Catalog\n" +
-                "\n" +
                 "connectionDetails:\n" +
-                "  \n" +
                 "  PETSDB::LOCAL: !<JDBCConnectionDetails>\n" +
                 "    userName: user1\n" +
                 "    password: pwd\n" +
@@ -120,7 +104,6 @@ public void test_parse_system_description() throws ModelStoreException {
                 "    - OWNERS\n" +
                 "    - PETS\n" +
                 "    - VETS\n" +
-                "\n" +
                 "  PRODUCTSDB::LOCAL: !<JDBCConnectionDetails>\n" +
                 "    userName: user1\n" +
                 "    password: pwd\n" +
@@ -129,7 +112,19 @@ public void test_parse_system_description() throws ModelStoreException {
                 "    - CUSTOMER\n" +
                 "    - PRODUCT\n" +
                 "    - SUPPLIER\n" +
-                "    \n";
+                "environments:\n" +
+                "- key: LOCAL\n" +
+                "  name: Local\n" +
+                "- key: DEV\n" +
+                "  name: Develop\n" +
+                "subSystems:\n" +
+                "- key: PETSDB\n" +
+                "  type: jdbc\n" +
+                "  name: Pets Database\n" +
+                "- key: PRODUCTSDB\n" +
+                "  type: jdbc\n" +
+                "  name: Product Catalog\n"
+                ;
 
         SystemDescription dbSystemDescription2 = (SystemDescription) yamlModelSerializer.loadModel(content, SystemDescription.class);
 

diff --git a/drift-sample/drift-sample-springboot/src/main/java/com/acme/AcmeModelStoreFactory.java b/drift-sample/drift-sample-springboot/src/main/java/com/acme/AcmeModelStoreFactory.java
@@ -52,7 +52,7 @@ public Class<?> getObjectType() {
 
     @Override
     public void destroy() {
-        if (luceneMetaDataStorage != null) luceneMetaDataStorage.shutdown();
+        if (luceneMetaDataStorage != null) luceneMetaDataStorage.shutDown();
     }
 
 }