Allow doc-values only search on geo_point fields (#83395)

Similar to #82409, but for geo_point fields.

Allows searching on geo_point fields when those fields are not indexed (index: false) but just doc values are enabled.

Also adds distance feature query support for date fields (bringing date field to feature parity with runtime fields)

This enables searches on archive data, which has access to doc values but not index structures. When combined with
searchable snapshots, it allows downloading only data for a given (doc value) field to quickly filter down to a select set
of documents.

Relates #81210 and #52728

Author: ywelsch

docs/changelog/83395.yaml

@@ -0,0 +1,5 @@
+pr: 83395
+summary: Allow doc-values only search on geo_point fields
+area: Mapping
+type: enhancement
+issues: []

docs/reference/mapping/params/doc-values.asciidoc

@@ -18,7 +18,7 @@ sorting and aggregations. Doc values are supported on almost all field types,
 with the __notable exception of `text` and `annotated_text` fields__.
 
 <<number,Numeric types>>, <<date,date types>>, the <<boolean,boolean type>>,
-the <<ip,ip type>> and the <<keyword,keyword type>>
+<<ip,ip type>>, <<geo-point,geo_point type>> and the <<keyword,keyword type>>
 can also be queried using term or range-based queries
 when they are not <<mapping-index,indexed>> but only have doc values enabled.
 Query performance on doc values is much slower than on index structures, but

docs/reference/mapping/types/geo-point.asciidoc

@@ -137,7 +137,9 @@ The following parameters are accepted by `geo_point` fields:
 
 <<mapping-index,`index`>>::
 
-    Should the field be searchable? Accepts `true` (default) and `false`.
+    Should the field be quickly searchable? Accepts `true` (default) and
+    `false`. Fields that only have <<doc-values,`doc_values`>>
+    enabled can still be queried, albeit slower.
 
 <<null-value,`null_value`>>::
 

docs/reference/query-dsl.asciidoc

@@ -33,7 +33,8 @@ the stability of the cluster. Those queries can be categorised as follows:
 
 * Queries that need to do linear scans to identify matches:
 ** <<query-dsl-script-query,`script` queries>>
-** queries on <<number,numeric>>, <<date,date>>, <<boolean,boolean>>, <<ip,ip>> or <<keyword,keyword>> fields
+** queries on <<number,numeric>>, <<date,date>>, <<boolean,boolean>>, <<ip,ip>>,
+   <<geo-point,geo_point>> or <<keyword,keyword>> fields
    that are not indexed but have <<doc-values,doc values>> enabled
 
 * Queries that have a high up-front cost:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml

@@ -95,6 +95,9 @@ setup:
                   non_indexed_ip:
                     type:     ip
                     index:    false
+                  non_indexed_geo_point:
+                    type:     geo_point
+                    index:    false
                   geo:
                     type:     keyword
                   object:
@@ -270,6 +273,19 @@ setup:
 
   - match: {fields.non_indexed_ip.ip.searchable:                          true}
 
+---
+"Field caps for geo_point field with only doc values":
+  - skip:
+      version: " - 8.0.99"
+      reason: "doc values search was added in 8.1.0"
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: non_indexed_geo_point
+
+  - match: {fields.non_indexed_geo_point.geo_point.searchable:            true}
+
+
 ---
 "Get object and nested field caps":
 

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml

@@ -45,6 +45,9 @@ setup:
               ip:
                 type: ip
                 index: false
+              geo_point:
+                type: geo_point
+                index: false
 
   - do:
       index:
@@ -62,6 +65,7 @@ setup:
           keyword: "key1"
           boolean: "false"
           ip: "192.168.0.1"
+          geo_point: [13.5, 34.89]
 
   - do:
       index:
@@ -79,6 +83,7 @@ setup:
           keyword: "key2"
           boolean: "true"
           ip: "192.168.0.2"
+          geo_point : [-63.24, 31.0]
 
   - do:
       indices.refresh: {}
@@ -236,6 +241,27 @@ setup:
         body: { query: { range: { date: { gte: "2017/01/01" } } } }
   - length:   { hits.hits: 2  }
 
+---
+"Test distance_feature query on date field where only doc values are enabled":
+  - skip:
+      version: " - 8.0.99"
+      reason: "doc values search was added in 8.1.0"
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            bool:
+              should:
+                distance_feature:
+                  field: "date"
+                  pivot: "7d"
+                  origin: "now"
+  - length:   { hits.hits: 2  }
+  - match: {hits.hits.0._id: "2" }
+  - match: {hits.hits.1._id: "1" }
+
 ---
 "Test match query on keyword field where only doc values are enabled":
 
@@ -316,3 +342,42 @@ setup:
         index: test
         body: { query: { range: { ip: { gte: "192.168.0.1" } } } }
   - length:   { hits.hits: 2  }
+
+---
+"Test geo shape query on geo_point field where only doc values are enabled":
+  - skip:
+      version: " - 8.0.99"
+      reason: "doc values search was added in 8.1.0"
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            geo_shape:
+              geo_point:
+                shape:
+                  type: "envelope"
+                  coordinates: [ [ -70, 32 ], [ -50, 30 ] ]
+  - match: {hits.total.value: 1}
+
+---
+"Test distance_feature query on geo_point field where only doc values are enabled":
+  - skip:
+      version: " - 8.0.99"
+      reason: "doc values search was added in 8.1.0"
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            bool:
+              should:
+                distance_feature:
+                  field: geo_point
+                  pivot: "1000km"
+                  origin: [0.0, 0.0]
+  - length:   { hits.hits: 2  }
+  - match: {hits.hits.0._id: "1" }
+  - match: {hits.hits.1._id: "2" }

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -42,10 +42,12 @@
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.field.DateMillisDocValuesField;
 import org.elasticsearch.script.field.DateNanosDocValuesField;
+import org.elasticsearch.script.field.SortedNumericDocValuesLongFieldScript;
 import org.elasticsearch.script.field.ToScriptField;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.runtime.LongScriptFieldDistanceFeatureQuery;
 
 import java.io.IOException;
 import java.text.NumberFormat;
@@ -86,6 +88,11 @@ public long convert(Instant instant) {
                 return instant.toEpochMilli();
             }
 
+            @Override
+            public long convert(TimeValue timeValue) {
+                return timeValue.millis();
+            }
+
             @Override
             public Instant toInstant(long value) {
                 return Instant.ofEpochMilli(value);
@@ -105,18 +112,18 @@ public long roundDownToMillis(long value) {
             public long roundUpToMillis(long value) {
                 return value;
             }
-
-            @Override
-            protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) {
-                return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getMillis());
-            }
         },
         NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS, DateNanosDocValuesField::new) {
             @Override
             public long convert(Instant instant) {
                 return toLong(instant);
             }
 
+            @Override
+            public long convert(TimeValue timeValue) {
+                return timeValue.nanos();
+            }
+
             @Override
             public Instant toInstant(long value) {
                 return DateUtils.toInstant(value);
@@ -141,11 +148,6 @@ public long roundUpToMillis(long value) {
                     return DateUtils.toMilliSeconds(value - 1L) + 1L;
                 }
             }
-
-            @Override
-            protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) {
-                return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos());
-            }
         };
 
         private final String type;
@@ -180,6 +182,11 @@ ToScriptField<SortedNumericDocValues> getDefaultToScriptField() {
          */
         public abstract Instant toInstant(long value);
 
+        /**
+         * Convert an {@linkplain TimeValue} into a long value in this resolution.
+         */
+        public abstract long convert(TimeValue timeValue);
+
         /**
          * Decode the points representation of this field as milliseconds.
          */
@@ -203,8 +210,6 @@ public static Resolution ofOrdinal(int ord) {
             }
             throw new IllegalArgumentException("unknown resolution ordinal [" + ord + "]");
         }
-
-        protected abstract Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot);
     }
 
     private static DateFieldMapper toType(FieldMapper in) {
@@ -596,10 +601,22 @@ public static long parseToLong(
 
         @Override
         public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) {
+            failIfNotIndexedNorDocValuesFallback(context);
             long originLong = parseToLong(origin, true, null, null, context::nowInMillis);
             TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot");
+            long pivotLong = resolution.convert(pivotTime);
             // As we already apply boost in AbstractQueryBuilder::toQuery, we always passing a boost of 1.0 to distanceFeatureQuery
-            return resolution.distanceFeatureQuery(name(), 1.0f, originLong, pivotTime);
+            if (isIndexed()) {
+                return LongPoint.newDistanceFeatureQuery(name(), 1.0f, originLong, pivotLong);
+            } else {
+                return new LongScriptFieldDistanceFeatureQuery(
+                    new Script(""),
+                    ctx -> new SortedNumericDocValuesLongFieldScript(name(), context.lookup(), ctx),
+                    name(),
+                    originLong,
+                    pivotLong
+                );
+            }
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -37,9 +37,11 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.field.GeoPointDocValuesField;
+import org.elasticsearch.script.field.SortedNumericDocValuesLongFieldScript;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.support.MapXContentParser;
@@ -268,6 +270,11 @@ public String typeName() {
             return CONTENT_TYPE;
         }
 
+        @Override
+        public boolean isSearchable() {
+            return isIndexed() || hasDocValues();
+        }
+
         @Override
         protected Function<List<GeoPoint>, List<Object>> getFormatter(String format) {
             return GEO_FORMATTER_FACTORY.getFormatter(format, p -> new Point(p.getLon(), p.getLat()));
@@ -284,6 +291,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
 
         @Override
         public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relation, SearchExecutionContext context) {
+            failIfNotIndexedNorDocValuesFallback(context);
             final LatLonGeometry[] luceneGeometries = GeoShapeUtils.toLuceneGeometry(fieldName, context, shape, relation);
             if (luceneGeometries.length == 0) {
                 return new MatchNoDocsQuery();
@@ -296,10 +304,15 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat
             } else {
                 luceneRelation = relation.getLuceneRelation();
             }
-            Query query = LatLonPoint.newGeometryQuery(fieldName, luceneRelation, luceneGeometries);
-            if (hasDocValues()) {
-                Query dvQuery = LatLonDocValuesField.newSlowGeometryQuery(fieldName, luceneRelation, luceneGeometries);
-                query = new IndexOrDocValuesQuery(query, dvQuery);
+            Query query;
+            if (isIndexed()) {
+                query = LatLonPoint.newGeometryQuery(fieldName, luceneRelation, luceneGeometries);
+                if (hasDocValues()) {
+                    Query dvQuery = LatLonDocValuesField.newSlowGeometryQuery(fieldName, luceneRelation, luceneGeometries);
+                    query = new IndexOrDocValuesQuery(query, dvQuery);
+                }
+            } else {
+                query = LatLonDocValuesField.newSlowGeometryQuery(fieldName, luceneRelation, luceneGeometries);
             }
             return query;
         }
@@ -312,6 +325,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
 
         @Override
         public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) {
+            failIfNotIndexedNorDocValuesFallback(context);
             GeoPoint originGeoPoint;
             if (origin instanceof GeoPoint) {
                 originGeoPoint = (GeoPoint) origin;
@@ -326,8 +340,19 @@ public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionCo
                 );
             }
             double pivotDouble = DistanceUnit.DEFAULT.parse(pivot, DistanceUnit.DEFAULT);
-            // As we already apply boost in AbstractQueryBuilder::toQuery, we always passing a boost of 1.0 to distanceFeatureQuery
-            return LatLonPoint.newDistanceFeatureQuery(name(), 1.0f, originGeoPoint.lat(), originGeoPoint.lon(), pivotDouble);
+            if (isIndexed()) {
+                // As we already apply boost in AbstractQueryBuilder::toQuery, we always passing a boost of 1.0 to distanceFeatureQuery
+                return LatLonPoint.newDistanceFeatureQuery(name(), 1.0f, originGeoPoint.lat(), originGeoPoint.lon(), pivotDouble);
+            } else {
+                return new GeoPointScriptFieldDistanceFeatureQuery(
+                    new Script(""),
+                    ctx -> new SortedNumericDocValuesLongFieldScript(name(), context.lookup(), ctx),
+                    name(),
+                    originGeoPoint.lat(),
+                    originGeoPoint.lon(),
+                    pivotDouble
+                );
+            }
         }
     }