Enable Support for Multiple Update Graphs (#3506)

* Refactor UpdateGraphProcessor into an UpdateGraph interface and PeriodicUpdateGraph implementation * Add UpdateGraph to ExecutionContext * Record UpdateGraph in many engine objects for efficient clock or lock access, and to allow operation safety-testing * Update the Python interface ugp.py to update_graph.py * Fix upstream QueryTableTest bug related to liveness reference counting around getRecord --------- Co-authored-by: Ryan Caudy <[email protected]>
deephaven · Jun 3, 2023 · ec85c5b · ec85c5b
1 parent c6de619
commit ec85c5b
Show file tree

Hide file tree

Showing 498 changed files with 8,857 additions and 6,911 deletions.
diff --git a/BenchmarkSupport/src/test/java/io/deephaven/benchmarking/impl/TestTableGeneration.java b/BenchmarkSupport/src/test/java/io/deephaven/benchmarking/impl/TestTableGeneration.java
@@ -4,7 +4,6 @@
 package io.deephaven.benchmarking.impl;
 
 import io.deephaven.engine.table.Table;
-import io.deephaven.engine.util.TableTools;
 import io.deephaven.engine.testutil.QueryTableTestBase;
 import io.deephaven.benchmarking.BenchmarkTable;
 import io.deephaven.benchmarking.BenchmarkTableBuilder;

diff --git a/Generators/src/main/java/io/deephaven/libs/GroovyStaticImportGenerator.java b/Generators/src/main/java/io/deephaven/libs/GroovyStaticImportGenerator.java
@@ -181,7 +181,7 @@ private GroovyStaticImportGenerator(final String[] imports, Collection<Predicate
         this.skips = skips;
 
         for (String imp : imports) {
-            Class<?> c = Class.forName(imp);
+            Class<?> c = Class.forName(imp, false, Thread.currentThread().getContextClassLoader());
             log.info("Processing class: " + c);
 
             for (Method m : c.getMethods()) {

diff --git a/Generators/src/main/java/io/deephaven/plot/util/GenerateFigureImmutable.java b/Generators/src/main/java/io/deephaven/plot/util/GenerateFigureImmutable.java
@@ -61,7 +61,7 @@ private GenerateFigureImmutable(final boolean isInterface, final String[] import
         this.functionNamer = functionNamer == null ? JavaFunction::getMethodName : functionNamer;
 
         for (final String imp : interfaces) {
-            final Class<?> c = Class.forName(imp);
+            final Class<?> c = Class.forName(imp, false, Thread.currentThread().getContextClassLoader());
             log.info("Processing class: " + c);
 
             for (final Method m : c.getMethods()) {
@@ -892,7 +892,7 @@ private Map<String, TreeSet<GroovyStaticImportGenerator.JavaFunction>> commonSig
 
         final Set<GroovyStaticImportGenerator.JavaFunction> functionSet = new HashSet<>();
         for (String iface : interfaces) {
-            final Class<?> c = Class.forName(iface);
+            final Class<?> c = Class.forName(iface, false, Thread.currentThread().getContextClassLoader());
             log.info("Processing class: " + c);
 
             for (final java.lang.reflect.Method m : c.getMethods()) {

diff --git a/Generators/src/main/java/io/deephaven/plot/util/GenerateMultiSeries.java b/Generators/src/main/java/io/deephaven/plot/util/GenerateMultiSeries.java
@@ -46,9 +46,11 @@ public static void main(String[] args) throws ClassNotFoundException, IOExceptio
         }
 
         final Set<Method> skip = new HashSet<>();
-        skip.add(Class.forName("io.deephaven.plot.datasets.DataSeries").getMethod("pointSize", int.class));
-        skip.add(Class.forName("io.deephaven.plot.datasets.DataSeries").getMethod("pointSize", double.class));
-        skip.add(Class.forName("io.deephaven.plot.datasets.DataSeries").getMethod("pointSize", long.class));
+        final Class<?> dataSeriesClass = Class.forName("io.deephaven.plot.datasets.DataSeries", false,
+                Thread.currentThread().getContextClassLoader());
+        skip.add(dataSeriesClass.getMethod("pointSize", int.class));
+        skip.add(dataSeriesClass.getMethod("pointSize", double.class));
+        skip.add(dataSeriesClass.getMethod("pointSize", long.class));
 
         new Generator("io.deephaven.plot.datasets.multiseries.MultiSeries",
                 "DataSeriesInternal",
@@ -170,7 +172,7 @@ static class Generator {
             this.isTransform = isTransform;
             this.isSwappable = isSwappable;
             this.interfaces = interfaces;
-            output = Class.forName(outputClass);
+            output = Class.forName(outputClass, false, Thread.currentThread().getContextClassLoader());
 
             final int mod = output.getModifiers();
             isInterface = Modifier.isInterface(mod);
@@ -241,7 +243,7 @@ private String generateClasses(final Set<Method> skip) throws ClassNotFoundExcep
             final List<GroovyStaticImportGenerator.JavaFunction> sortedMethods = new ArrayList<>();
             final List<GroovyStaticImportGenerator.JavaFunction> methodsWithFunctionParameter = new ArrayList<>();
             for (final String clazz : interfaces) {
-                final Class dataseries = Class.forName(clazz);
+                final Class dataseries = Class.forName(clazz, false, Thread.currentThread().getContextClassLoader());
                 final Method[] methods = Arrays.stream(dataseries.getMethods())
                         .filter(m -> !skip.contains(m))
                         .toArray(Method[]::new);
@@ -561,7 +563,8 @@ private String getFigureFunctionInput(final String returnClass,
                     : "getPartitionedTableHandle().getTable(), ");
 
             if (function.getMethodName().equals("pointColorByY")) {
-                final Class c = Class.forName("io.deephaven.plot.datasets.multiseries." + returnClass);
+                final Class c = Class.forName("io.deephaven.plot.datasets.multiseries." + returnClass, false,
+                        Thread.currentThread().getContextClassLoader());
                 final Method[] methods = Arrays.stream(c.getDeclaredMethods())
                         .filter(m -> m.getName().equals(tableMethodName))
                         .filter(m -> m.getParameterTypes().length > 0 && m.getParameterTypes()[0].equals(Table.class))
@@ -586,7 +589,8 @@ private String getFigureFunctionInput(final String returnClass,
                 return code.append(", multiSeriesKey), this").toString();
             }
 
-            final Class c = Class.forName(function.getClassName());
+            final Class c = Class.forName(function.getClassName(), false,
+                    Thread.currentThread().getContextClassLoader());
             final Method[] methods = Arrays.stream(c.getMethods())
                     .filter(m -> m.getName().equals(tableMethodName))
                     .filter(m -> m.getParameterTypes().length > 0 && m.getParameterTypes()[0].equals(Table.class))

diff --git a/Generators/src/main/java/io/deephaven/plot/util/GeneratePlottingConvenience.java b/Generators/src/main/java/io/deephaven/plot/util/GeneratePlottingConvenience.java
@@ -49,7 +49,7 @@ private GeneratePlottingConvenience(final String[] staticImports, final String[]
             final int lastDot = staticImport.lastIndexOf(".");
             final String classPath = staticImport.substring(0, lastDot);
             final String methodName = staticImport.substring(lastDot + 1);
-            final Class<?> c = Class.forName(classPath);
+            final Class<?> c = Class.forName(classPath, false, Thread.currentThread().getContextClassLoader());
             log.info("Processing static class: " + c);
 
             final Method[] methods = Arrays.stream(c.getMethods()).filter(
@@ -64,7 +64,7 @@ private GeneratePlottingConvenience(final String[] staticImports, final String[]
         }
 
         for (final String imp : imports) {
-            final Class<?> c = Class.forName(imp);
+            final Class<?> c = Class.forName(imp, false, Thread.currentThread().getContextClassLoader());
             log.info("Processing class: " + c);
 
             for (final Method m : c.getMethods()) {

diff --git a/Generators/src/main/java/io/deephaven/plot/util/GeneratePyV2FigureAPI.java b/Generators/src/main/java/io/deephaven/plot/util/GeneratePyV2FigureAPI.java
@@ -123,7 +123,7 @@ public static void main(String[] args) throws ClassNotFoundException, IOExceptio
      * @throws ClassNotFoundException JCLASS is not found
      */
     public static Map<Key, ArrayList<JavaFunction>> getMethodSignatures() throws ClassNotFoundException {
-        final Class<?> c = Class.forName(JCLASS);
+        final Class<?> c = Class.forName(JCLASS, false, Thread.currentThread().getContextClassLoader());
         final Map<Key, ArrayList<JavaFunction>> signatures = new TreeMap<>();
 
         for (final Method m : c.getMethods()) {

diff --git a/Integrations/build.gradle b/Integrations/build.gradle
@@ -18,6 +18,8 @@ dependencies {
     implementation project(':Configuration')
     implementation project(':log-factory')
 
+    testImplementation project(':engine-test-utils')
+
     testRuntimeOnly project(':log-to-slf4j')
     // add configs, and some runtime dependencies to test classpaths
     testRuntimeOnly project(':configs')

diff --git a/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java b/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java
@@ -6,9 +6,10 @@
 import io.deephaven.base.FileUtils;
 import io.deephaven.base.verify.Assert;
 import io.deephaven.configuration.Configuration;
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.exceptions.CancellationException;
 import io.deephaven.engine.context.QueryScope;
-import io.deephaven.engine.updategraph.UpdateGraphProcessor;
+import io.deephaven.engine.updategraph.UpdateGraph;
 import io.deephaven.engine.util.AbstractScriptSession;
 import io.deephaven.engine.util.PythonEvaluator;
 import io.deephaven.engine.util.PythonEvaluatorJpy;
@@ -69,17 +70,20 @@ public class PythonDeephavenSession extends AbstractScriptSession<PythonSnapshot
     /**
      * Create a Python ScriptSession.
      *
+     * @param updateGraph the default update graph to install for the repl
      * @param objectTypeLookup the object type lookup
      * @param listener an optional listener that will be notified whenever the query scope changes
      * @param runInitScripts if init scripts should be executed
      * @param pythonEvaluator
      * @throws IOException if an IO error occurs running initialization scripts
      */
     public PythonDeephavenSession(
-            ObjectTypeLookup objectTypeLookup, @Nullable final Listener listener, boolean runInitScripts,
-            PythonEvaluatorJpy pythonEvaluator)
-            throws IOException {
-        super(objectTypeLookup, listener);
+            final UpdateGraph updateGraph,
+            final ObjectTypeLookup objectTypeLookup,
+            @Nullable final Listener listener,
+            final boolean runInitScripts,
+            final PythonEvaluatorJpy pythonEvaluator) throws IOException {
+        super(updateGraph, objectTypeLookup, listener);
 
         evaluator = pythonEvaluator;
         scope = pythonEvaluator.getScope();
@@ -108,8 +112,9 @@ public PythonDeephavenSession(
      * Creates a Python "{@link ScriptSession}", for use where we should only be reading from the scope, such as an
      * IPython kernel session.
      */
-    public PythonDeephavenSession(PythonScope<?> scope) {
-        super(NoOp.INSTANCE, null);
+    public PythonDeephavenSession(
+            final UpdateGraph updateGraph, final PythonScope<?> scope) {
+        super(updateGraph, NoOp.INSTANCE, null);
         this.scope = (PythonScope<PyObject>) scope;
         try (final SafeCloseable ignored = executionContext.open()) {
             this.module = (PythonScriptSessionModule) PyModule.importModule("deephaven.server.script_session")
@@ -186,9 +191,8 @@ public void popScope() {
     protected void evaluate(String command, String scriptName) {
         log.info().append("Evaluating command: " + command).endl();
         try {
-            UpdateGraphProcessor.DEFAULT.exclusiveLock().doLockedInterruptibly(() -> {
-                evaluator.evalScript(command);
-            });
+            ExecutionContext.getContext().getUpdateGraph().exclusiveLock()
+                    .doLockedInterruptibly(() -> evaluator.evalScript(command));
         } catch (InterruptedException e) {
             throw new CancellationException(e.getMessage() != null ? e.getMessage() : "Query interrupted", e);
         }

diff --git a/Integrations/src/test/java/io/deephaven/integrations/learn/ComputerTest.java b/Integrations/src/test/java/io/deephaven/integrations/learn/ComputerTest.java
@@ -8,19 +8,21 @@
 import io.deephaven.engine.rowset.WritableRowSet;
 import io.deephaven.engine.table.impl.InMemoryTable;
 import io.deephaven.engine.table.ColumnSource;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import io.deephaven.engine.testutil.junit4.EngineCleanup;
+import org.junit.*;
 
 import java.util.Objects;
 import java.util.function.Function;
 
 public class ComputerTest {
 
-    private static InMemoryTable table;
+    @Rule
+    public final EngineCleanup framework = new EngineCleanup();
 
-    @BeforeClass
-    public static void createTable() {
+    private InMemoryTable table;
+
+    @Before
+    public void createTable() {
         table = new InMemoryTable(
                 new String[] {"Column1", "Column2", "Column3"},
                 new Object[] {

diff --git a/Integrations/src/test/java/io/deephaven/integrations/learn/FutureTest.java b/Integrations/src/test/java/io/deephaven/integrations/learn/FutureTest.java
@@ -8,19 +8,21 @@
 import io.deephaven.engine.rowset.WritableRowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.InMemoryTable;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import io.deephaven.engine.testutil.junit4.EngineCleanup;
+import org.junit.*;
 
 import java.util.Objects;
 import java.util.function.Function;
 
 public class FutureTest {
 
-    private static InMemoryTable table;
+    @Rule
+    public final EngineCleanup framework = new EngineCleanup();
 
-    @BeforeClass
-    public static void createTable() {
+    private InMemoryTable table;
+
+    @Before
+    public void createTable() {
         table = new InMemoryTable(
                 new String[] {"Column1", "Column2", "Column3"},
                 new Object[] {
@@ -41,7 +43,7 @@ private static Input[] createInputs(Function<Object[], Object> gatherFunc) {
         return createInputs(gatherFunc, gatherFunc);
     }
 
-    private static Future createFuture(Function<Object[], Object> modelFunc, Input[] inputs, int batchSize) {
+    private Future createFuture(Function<Object[], Object> modelFunc, Input[] inputs, int batchSize) {
         return new Future(modelFunc, inputs,
                 new ColumnSource[][] {
                         table.view("Column1", "Column2").getColumnSources()

diff --git a/Integrations/src/test/java/io/deephaven/integrations/learn/InputTest.java b/Integrations/src/test/java/io/deephaven/integrations/learn/InputTest.java
@@ -6,19 +6,21 @@
 import io.deephaven.engine.table.impl.InMemoryTable;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.api.util.NameValidator;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import io.deephaven.engine.testutil.junit4.EngineCleanup;
+import org.junit.*;
 
 import java.util.Arrays;
 import java.util.function.Function;
 
 public class InputTest {
 
-    private static InMemoryTable table;
+    @Rule
+    public final EngineCleanup framework = new EngineCleanup();
 
-    @BeforeClass
-    public static void createTable() {
+    private InMemoryTable table;
+
+    @Before
+    public void createTable() {
         table = new InMemoryTable(
                 new String[] {"Column1", "Column2", "Column3"},
                 new Object[] {

diff --git a/Integrations/src/test/java/io/deephaven/integrations/learn/ScattererTest.java b/Integrations/src/test/java/io/deephaven/integrations/learn/ScattererTest.java
@@ -4,19 +4,20 @@
 package io.deephaven.integrations.learn;
 
 import io.deephaven.engine.table.impl.InMemoryTable;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import io.deephaven.engine.testutil.junit4.EngineCleanup;
+import org.junit.*;
 
 import java.util.function.Function;
 
 public class ScattererTest {
 
-    private static InMemoryTable table;
+    @Rule
+    public final EngineCleanup framework = new EngineCleanup();
 
+    private InMemoryTable table;
 
-    @BeforeClass
-    public static void setup() {
+    @Before
+    public void setup() {
         table = new InMemoryTable(
                 new String[] {"Column1", "Column2", "Column3"},
                 new Object[] {

diff --git a/Integrations/src/test/java/io/deephaven/integrations/learn/gather/NumPyTest.java b/Integrations/src/test/java/io/deephaven/integrations/learn/gather/NumPyTest.java
@@ -6,15 +6,17 @@
 import io.deephaven.engine.rowset.RowSequence;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.InMemoryTable;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import io.deephaven.engine.testutil.junit4.EngineCleanup;
+import org.junit.*;
+
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
 public class NumPyTest {
 
-    private static InMemoryTable table;
+    @Rule
+    public final EngineCleanup framework = new EngineCleanup();
+
     private static final String[] boolColNames = {"bool1", "bool2"};
     private static final boolean[][] boolData = {
             new boolean[] {true, true, false, false},
@@ -69,12 +71,14 @@ public class NumPyTest {
             doubleData[0], doubleData[1]
     };
 
-    @BeforeClass
-    public static void setup() {
+    private InMemoryTable table;
+
+    @Before
+    public void setup() {
         table = new InMemoryTable(columnNames, columnData);
     }
 
-    public static ColumnSource<?>[] getColSet(final String[] colNames) {
+    public ColumnSource<?>[] getColSet(final String[] colNames) {
         ColumnSource<?>[] rst = new ColumnSource[2];
 
         for (int i = 0; i < 2; i++) {