Add support for non-aggregated rows & other feedback improvements

foster33 · foster33 · commit faac0315ed7a · 2025-10-23T20:34:10.000Z
diff --git a/core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java b/core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java
@@ -5,6 +5,7 @@
 import java.io.IOException;
 import java.nio.charset.CharacterCodingException;
 import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Collection;
@@ -40,6 +41,7 @@
 import org.apache.accumulo.core.data.PartialKey;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.FirstEntryInRowIterator;
 import org.apache.accumulo.core.iterators.ValueFormatException;
 import org.apache.accumulo.core.iterators.user.RegExFilter;
 import org.apache.accumulo.core.iterators.user.SummingCombiner;
@@ -1714,6 +1716,7 @@ public Set<String> getMissingFieldsInDateRange(Set<String> fields, Set<String> d
         fields = Sets.difference(fields, specialFields);
         Set<Range> ranges = createExactFieldCountRanges(fields);
         StringBuilder dataTypeRegex = new StringBuilder();
+        List<IteratorSetting> settings = new ArrayList<>();
 
         if (ranges.isEmpty()) {
             return Collections.emptySet();
@@ -1735,22 +1738,55 @@ public Set<String> getMissingFieldsInDateRange(Set<String> fields, Set<String> d
         }
 
         try (BatchScanner bs = ScannerHelper.createBatchScanner(client, getMetadataTableName(), getAuths(), fields.size())) {
-            IteratorSetting regexIter = new IteratorSetting(50, "regexFilter", RegExFilter.class);
+            settings.add(new IteratorSetting(51, "FirstEntryInRow", FirstEntryInRowIterator.class));
+            settings.add(new IteratorSetting(50, "regexFilter", RegExFilter.class));
             if (!dataTypeRegex.toString().isEmpty()) {
-                regexIter.addOption(RegExFilter.COLQ_REGEX, dataTypeRegex.toString());
+                for (IteratorSetting setting : settings) {
+                    if (setting.getName().equals("regexFilter")) {
+                        setting.addOption(RegExFilter.COLQ_REGEX, dataTypeRegex.toString());
+                        break;
+                    }
+                }
             }
             bs.setRanges(ranges);
-            bs.addScanIterator(regexIter);
+            for (IteratorSetting setting : settings) {
+                bs.addScanIterator(setting);
+            }
 
             for (Entry<Key,Value> entry : bs) {
+                Text colq = entry.getKey().getColumnQualifier();
+                int colqIndex = colq.find(NULL_BYTE);
+
+                String remainder;
                 try {
-                    DateFrequencyMap map = new DateFrequencyMap(entry.getValue().get());
-                    if (!map.subMap(beginDate, endDate).isEmpty()) {
-                        foundFields.add(entry.getKey().getRow().toString());
+                    remainder = Text.decode(colq.getBytes(), colqIndex + 1, colq.getLength() - (colqIndex + 1));
+                } catch (CharacterCodingException e) {
+                    log.warn("Could not deserialize colqual: {} ", entry.getKey());
+                    continue;
+                }
+                if (remainder.equals(FrequencyMetadataAggregator.AGGREGATED)) {
+                    // This is an aggregated entry.
+                    try {
+                        DateFrequencyMap map = new DateFrequencyMap(entry.getValue().get());
+                        if (!map.subMap(beginDate, endDate).isEmpty()) {
+                            foundFields.add(entry.getKey().getRow().toString());
+                        }
+                    } catch (IOException e) {
+                        log.error("Failed to convert Value to DateFrequencyMap", e);
+                    }
+                } else {
+                    // This is an entry with a count for a single date.
+                    try {
+                        Date date = DateHelper.parse(remainder);
+                        // Add the field if we fall within beginDate and endDate, inclusively.
+                        if (date.compareTo(DateHelper.parse(beginDate)) >= 0 && date.compareTo(DateHelper.parse(endDate)) <= 0) {
+                            foundFields.add(entry.getKey().getRow().toString());
+                        }
+                    } catch (ValueFormatException e) {
+                        log.warn("Could not convert the Value to a long: {}", entry.getValue());
+                    } catch (DateTimeParseException e) {
+                        log.warn("Could not convert date string: {}", remainder);
                     }
-                } catch (IOException e) {
-                    log.trace("Could not convert the Value to a DateFrequencyMap: {}", entry.getValue());
-                    log.error("Failed to convert Value to DateFrequencyMap", e);
                 }
             }
         } catch (TableNotFoundException e) {
diff --git a/core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java b/core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java
@@ -392,4 +392,100 @@ void testMixedEntryFormats() {
             Assertions.assertEquals(DateHelper.parse("20200103"), helper.getEarliestOccurrenceOfFieldWithType("NAME", "maze", accumuloClient, null));
         }
     }
+
+    /**
+     * Tests for {@link MetadataHelper#getMissingFieldsInDateRange(Set, Set, String, String, Set)}.
+     */
+    @Nested
+    public class GetMissingFieldsInDateRangeTest {
+        /**
+         * Test against a table that has only non-aggregated entries as matches.
+         */
+        @Test
+        void testNonAggregatedEntriesOnly() throws TableNotFoundException {
+            givenNonAggregatedFrequencyRows("NAME", COLF_F, "csv", "20200103", "20200120", 1L);
+            givenNonAggregatedFrequencyRows("NAME", COLF_F, "wiki", "20200101", "20200120", 2L);
+            givenNonAggregatedFrequencyRows("NAME", COLF_F, "maze", "20200105", "20200120", 3L);
+            givenNonAggregatedFrequencyRows("NAME", COLF_F, "data", "20200107", "20200102", 3L);
+            givenNonAggregatedFrequencyRows("EVENT_DATE", COLF_F, "csv", "20200101", "20200120", 4L);
+            givenNonAggregatedFrequencyRows("EVENT_DATE", COLF_F, "wiki", "20200101", "20200120", 5L);
+            givenNonAggregatedFrequencyRows("EVENT_DATE", COLF_F, "maze", "20200101", "20200120", 6L);
+            writeMutations();
+
+            // No DataTypes
+            Assertions.assertEquals(Collections.emptySet(), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Collections.emptySet(), "20200101",
+                            "20200120", Collections.emptySet()));
+            // Using DataTypes
+            Assertions.assertEquals(Set.of("EVENT_DATE"),
+                            helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Set.of("data"), "20200101", "20200120", Collections.emptySet()));
+            // Fictitious field
+            Assertions.assertEquals(Set.of("FOO"), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE", "FOO"),
+                            Set.of("wiki", "data", "csv", "maze"), "20200101", "20200120", Collections.emptySet()));
+            // Missing because of date range
+            Assertions.assertEquals(Set.of("NAME", "EVENT_DATE"), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Set.of("wiki", "data"),
+                            "20190101", "20191231", Collections.emptySet()));
+        }
+
+        /**
+         * Test against a table that has only aggregated entries as matches.
+         */
+        @Test
+        void testAggregatedEntriesOnly() throws TableNotFoundException {
+            givenAggregatedFrequencyRow("NAME", COLF_F, "csv", createDateFrequencyMap("20200113", 1L, "20200115", 5L, "20200116", 3L));
+            givenAggregatedFrequencyRow("NAME", COLF_F, "wiki", createDateFrequencyMap("20200111", 1L, "20200112", 15L, "20200113", 3L));
+            givenAggregatedFrequencyRow("NAME", COLF_F, "maze", createDateFrequencyMap("20200102", 1L, "20200104", 55L, "20200105", 3L));
+            givenAggregatedFrequencyRow("NAME", COLF_F, "data", createDateFrequencyMap("20200101", 1L, "20200103", 3L));
+            givenAggregatedFrequencyRow("EVENT_DATE", COLF_F, "csv", createDateFrequencyMap("20200101", 2L, "20200102", 3L, "20200103", 4L));
+            givenAggregatedFrequencyRow("EVENT_DATE", COLF_F, "wiki", createDateFrequencyMap("20200101", 2L, "20200102", 3L, "20200103", 4L));
+            givenAggregatedFrequencyRow("EVENT_DATE", COLF_F, "maze", createDateFrequencyMap("20200101", 2L, "20200102", 3L, "20200103", 4L));
+            writeMutations();
+
+            // No DataTypes
+            Assertions.assertEquals(Collections.emptySet(), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Collections.emptySet(), "20200101",
+                            "20200120", Collections.emptySet()));
+            // Using DataTypes
+            Assertions.assertEquals(Set.of("EVENT_DATE"),
+                            helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Set.of("data"), "20200101", "20200120", Collections.emptySet()));
+            // Fictitious field
+            Assertions.assertEquals(Set.of("FOO"), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE", "FOO"),
+                            Set.of("wiki", "data", "csv", "maze"), "20200101", "20200120", Collections.emptySet()));
+            // Missing because of date range
+            Assertions.assertEquals(Set.of("NAME", "EVENT_DATE"), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Set.of("wiki", "data"),
+                            "20190101", "20191231", Collections.emptySet()));
+        }
+
+        /**
+         * Test against a table that has both aggregated and non-aggregated entries as matches.
+         */
+        @Test
+        void testMixedEntryFormats() throws TableNotFoundException {
+            givenAggregatedFrequencyRow("NAME", COLF_F, "csv", createDateFrequencyMap("20200111", 1L, "20200112", 5L, "20200113", 3L));
+            givenNonAggregatedFrequencyRows("NAME", COLF_F, "csv", "20200111", "20200120", 1L);
+            givenAggregatedFrequencyRow("NAME", COLF_F, "wiki", createDateFrequencyMap("20200111", 1L, "20200112", 15L, "20200113", 3L));
+            givenAggregatedFrequencyRow("NAME", COLF_F, "maze", createDateFrequencyMap("20200111", 1L, "20200112", 55L, "20200113", 3L));
+            givenNonAggregatedFrequencyRows("NAME", COLF_F, "maze", "20200103", "20200120", 3L);
+            givenAggregatedFrequencyRow("NAME", COLF_F, "data", createDateFrequencyMap("20200111", 1L, "20200113", 3L));
+            givenNonAggregatedFrequencyRows("NAME", COLF_F, "data", "20200101", "20200115", 3L);
+            givenAggregatedFrequencyRow("EVENT_DATE", COLF_F, "csv", createDateFrequencyMap("20200101", 2L, "20200102", 3L, "20200103", 4L));
+            givenAggregatedFrequencyRow("EVENT_DATE", COLF_F, "wiki", createDateFrequencyMap("20200101", 2L, "20200102", 3L, "20200103", 4L));
+            givenAggregatedFrequencyRow("EVENT_DATE", COLF_F, "maze", createDateFrequencyMap("20200101", 2L, "20200102", 3L, "20200103", 4L));
+            givenNonAggregatedFrequencyRows("EVENT_DATE", COLF_F, "csv", "20200101", "20200120", 4L);
+            givenNonAggregatedFrequencyRows("EVENT_DATE", COLF_F, "wiki", "20200101", "20200120", 5L);
+            givenNonAggregatedFrequencyRows("EVENT_DATE", COLF_F, "maze", "20200101", "20200120", 6L);
+            writeMutations();
+
+            // No DataTypes
+            Assertions.assertEquals(Collections.emptySet(), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Collections.emptySet(), "20200101",
+                            "20200120", Collections.emptySet()));
+            // Using DataTypes
+            Assertions.assertEquals(Set.of("EVENT_DATE"),
+                            helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Set.of("data"), "20200101", "20200120", Collections.emptySet()));
+            // Fictitious field
+            Assertions.assertEquals(Set.of("FOO"), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE", "FOO"),
+                            Set.of("wiki", "data", "csv", "maze"), "20200101", "20200120", Collections.emptySet()));
+            // Missing because of date range
+            Assertions.assertEquals(Set.of("NAME", "EVENT_DATE"), helper.getMissingFieldsInDateRange(Set.of("NAME", "EVENT_DATE"), Set.of("wiki", "data"),
+                            "20190101", "20191231", Collections.emptySet()));
+        }
+    }
 }
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromDateRangeVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromDateRangeVisitorTest.java
