From 90ed7034ce3b6ce32190518c635f89c33a7695a2 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 1 Feb 2022 15:44:31 +0100 Subject: [PATCH 1/8] Show how script-based queries can be used with doc values --- .../index/mapper/DateFieldMapper.java | 51 ++++++++++++++++++- .../index/mapper/DateScriptFieldType.java | 3 +- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index c000137df3074..5fab9ede94b12 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; @@ -42,6 +43,7 @@ import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.DateMillisDocValuesField; import org.elasticsearch.script.field.DateNanosDocValuesField; +import org.elasticsearch.script.field.LongDocValuesField; import org.elasticsearch.script.field.ToScriptField; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.FieldValues; @@ -367,6 +369,8 @@ public static final class DateFieldType extends MappedFieldType { protected final String nullValue; protected final FieldValues scriptValues; + protected final DateScriptFieldType docValueBasedScriptFieldType; + public DateFieldType( String name, boolean isIndexed, @@ -384,6 +388,49 @@ public DateFieldType( this.resolution = resolution; this.nullValue = nullValue; this.scriptValues = scriptValues; + docValueBasedScriptFieldType = new DateScriptFieldType(name, createDocValuesScriptFactory(), dateTimeFormatter, new Script(""), meta); + } + + public DateFieldScript.Factory createDocValuesScriptFactory() { + return new DateFieldScript.Factory() { + @Override + public DateFieldScript.LeafFactory newFactory(String field, Map params, + SearchLookup lookup, DateFormatter formatter) { + return ctx -> new DateFieldScript(field, params, lookup, formatter, ctx) { + + final LongDocValuesField longDocValuesField; + + { + try { + longDocValuesField = new LongDocValuesField(DocValues.getSortedNumeric(ctx.reader(), field), field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public void setDocument(int docID) { + try { + longDocValuesField.setNextDocId(docID); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public void execute() { + for (long value : longDocValuesField) { + emit(value); + } + } + }; + } + + @Override + public boolean isResultDeterministic() { + return true; + } + }; } public DateFieldType(String name) { @@ -520,7 +567,9 @@ public Query rangeQuery( query = new IndexOrDocValuesQuery(query, dvQuery); } } else { - query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = docValueBasedScriptFieldType.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, relation, timeZone, + forcedDateParser, context); + //query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); } if (hasDocValues() && context.indexSortedOnField(name())) { query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index 40ebc9fa9e59d..520397f5b2578 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -19,6 +19,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.fielddata.DateScriptFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.query.SearchExecutionContext; @@ -160,7 +161,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { } @Override - public DateScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier lookup) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier lookup) { return new DateScriptFieldData.Builder(name(), leafFactory(lookup.get()), Resolution.MILLISECONDS.getDefaultToScriptField()); } From d39d6638ec3aded9d84c008ca6317bced7b317bf Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 1 Feb 2022 17:27:01 +0100 Subject: [PATCH 2/8] distance feature query on date field --- .../test/search/390_doc_values_search.yml | 18 ++++ .../index/mapper/DateFieldMapper.java | 89 ++++++------------- ...SortedNumericDocValuesLongFieldScript.java | 52 +++++++++++ 3 files changed, 96 insertions(+), 63 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml index 323c521f4d128..a0552b7f079fe 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml @@ -236,6 +236,24 @@ setup: body: { query: { range: { date: { gte: "2017/01/01" } } } } - length: { hits.hits: 2 } +--- +"Test distance_feature query on date field where only doc values are enabled": + + - do: + search: + index: test + body: + query: + bool: + should: + distance_feature: + field: "date" + pivot: "7d" + origin: "now" + - length: { hits.hits: 2 } + - match: {hits.hits.0._id: "2" } + - match: {hits.hits.1._id: "1" } + --- "Test match query on keyword field where only doc values are enabled": diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 5fab9ede94b12..25544471d7ac3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -11,7 +11,6 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; @@ -43,11 +42,12 @@ import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.DateMillisDocValuesField; import org.elasticsearch.script.field.DateNanosDocValuesField; -import org.elasticsearch.script.field.LongDocValuesField; +import org.elasticsearch.script.field.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.script.field.ToScriptField; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.runtime.LongScriptFieldDistanceFeatureQuery; import java.io.IOException; import java.text.NumberFormat; @@ -88,6 +88,11 @@ public long convert(Instant instant) { return instant.toEpochMilli(); } + @Override + public long convert(TimeValue timeValue) { + return timeValue.millis(); + } + @Override public Instant toInstant(long value) { return Instant.ofEpochMilli(value); @@ -107,11 +112,6 @@ public long roundDownToMillis(long value) { public long roundUpToMillis(long value) { return value; } - - @Override - protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { - return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getMillis()); - } }, NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS, DateNanosDocValuesField::new) { @Override @@ -119,6 +119,11 @@ public long convert(Instant instant) { return toLong(instant); } + @Override + public long convert(TimeValue timeValue) { + return timeValue.nanos(); + } + @Override public Instant toInstant(long value) { return DateUtils.toInstant(value); @@ -143,11 +148,6 @@ public long roundUpToMillis(long value) { return DateUtils.toMilliSeconds(value - 1L) + 1L; } } - - @Override - protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { - return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos()); - } }; private final String type; @@ -182,6 +182,11 @@ ToScriptField getDefaultToScriptField() { */ public abstract Instant toInstant(long value); + /** + * Convert an {@linkplain TimeValue} into a long value in this resolution. + */ + public abstract long convert(TimeValue timeValue); + /** * Decode the points representation of this field as milliseconds. */ @@ -205,8 +210,6 @@ public static Resolution ofOrdinal(int ord) { } throw new IllegalArgumentException("unknown resolution ordinal [" + ord + "]"); } - - protected abstract Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot); } private static DateFieldMapper toType(FieldMapper in) { @@ -369,8 +372,6 @@ public static final class DateFieldType extends MappedFieldType { protected final String nullValue; protected final FieldValues scriptValues; - protected final DateScriptFieldType docValueBasedScriptFieldType; - public DateFieldType( String name, boolean isIndexed, @@ -388,49 +389,6 @@ public DateFieldType( this.resolution = resolution; this.nullValue = nullValue; this.scriptValues = scriptValues; - docValueBasedScriptFieldType = new DateScriptFieldType(name, createDocValuesScriptFactory(), dateTimeFormatter, new Script(""), meta); - } - - public DateFieldScript.Factory createDocValuesScriptFactory() { - return new DateFieldScript.Factory() { - @Override - public DateFieldScript.LeafFactory newFactory(String field, Map params, - SearchLookup lookup, DateFormatter formatter) { - return ctx -> new DateFieldScript(field, params, lookup, formatter, ctx) { - - final LongDocValuesField longDocValuesField; - - { - try { - longDocValuesField = new LongDocValuesField(DocValues.getSortedNumeric(ctx.reader(), field), field); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - } - - @Override - public void setDocument(int docID) { - try { - longDocValuesField.setNextDocId(docID); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - } - - @Override - public void execute() { - for (long value : longDocValuesField) { - emit(value); - } - } - }; - } - - @Override - public boolean isResultDeterministic() { - return true; - } - }; } public DateFieldType(String name) { @@ -567,9 +525,7 @@ public Query rangeQuery( query = new IndexOrDocValuesQuery(query, dvQuery); } } else { - query = docValueBasedScriptFieldType.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, relation, timeZone, - forcedDateParser, context); - //query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); } if (hasDocValues() && context.indexSortedOnField(name())) { query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); @@ -645,10 +601,17 @@ public static long parseToLong( @Override public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) { + failIfNotIndexedNorDocValuesFallback(context); long originLong = parseToLong(origin, true, null, null, context::nowInMillis); TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot"); + long pivotLong = resolution.convert(pivotTime); // As we already apply boost in AbstractQueryBuilder::toQuery, we always passing a boost of 1.0 to distanceFeatureQuery - return resolution.distanceFeatureQuery(name(), 1.0f, originLong, pivotTime); + if (isIndexed()) { + return LongPoint.newDistanceFeatureQuery(name(), 1.0f, originLong, pivotLong); + } else { + return new LongScriptFieldDistanceFeatureQuery(new Script(""), + ctx -> new SortedNumericDocValuesLongFieldScript(name(), context.lookup(), ctx), name(), originLong, pivotLong); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java b/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java new file mode 100644 index 0000000000000..00b7af4215e2f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.script.AbstractLongFieldScript; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.Map; + +public class SortedNumericDocValuesLongFieldScript extends AbstractLongFieldScript { + + final LongDocValuesField longDocValuesField; + + public SortedNumericDocValuesLongFieldScript(String fieldName, SearchLookup lookup, LeafReaderContext ctx) { + super(fieldName, Map.of(), lookup, ctx); + try { + longDocValuesField = new LongDocValuesField(DocValues.getSortedNumeric(ctx.reader(), fieldName), fieldName); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + protected void emitFromObject(Object v) { + throw new UnsupportedOperationException(); + } + + @Override + public void setDocument(int docID) { + try { + longDocValuesField.setNextDocId(docID); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public void execute() { + for (long value : longDocValuesField) { + emit(value); + } + } +} From 7d7de96bffc57291a8583e1d486f9b2d16a41aa0 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 1 Feb 2022 18:50:57 +0100 Subject: [PATCH 3/8] Add doc value only searches for geo_point --- .../mapping/params/doc-values.asciidoc | 2 +- .../mapping/types/geo-point.asciidoc | 4 +- docs/reference/query-dsl.asciidoc | 3 +- .../test/field_caps/10_basic.yml | 16 +++++++ .../test/search/390_doc_values_search.yml | 47 +++++++++++++++++++ .../index/mapper/DateFieldMapper.java | 9 +++- .../index/mapper/DateScriptFieldType.java | 3 +- .../index/mapper/GeoPointFieldMapper.java | 37 ++++++++++++--- 8 files changed, 108 insertions(+), 13 deletions(-) diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc index 1dc585b3975fd..0b4a2c7cf4b2f 100644 --- a/docs/reference/mapping/params/doc-values.asciidoc +++ b/docs/reference/mapping/params/doc-values.asciidoc @@ -18,7 +18,7 @@ sorting and aggregations. Doc values are supported on almost all field types, with the __notable exception of `text` and `annotated_text` fields__. <>, <>, the <>, -the <> and the <> +<>, <> and the <> can also be queried using term or range-based queries when they are not <> but only have doc values enabled. Query performance on doc values is much slower than on index structures, but diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index dd878aa595170..d6e23c2bcef70 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -137,7 +137,9 @@ The following parameters are accepted by `geo_point` fields: <>:: - Should the field be searchable? Accepts `true` (default) and `false`. + Should the field be quickly searchable? Accepts `true` (default) and + `false`. Fields that only have <> + enabled can still be queried, albeit slower. <>:: diff --git a/docs/reference/query-dsl.asciidoc b/docs/reference/query-dsl.asciidoc index fa9334f390ed1..8dd9c30e27c67 100644 --- a/docs/reference/query-dsl.asciidoc +++ b/docs/reference/query-dsl.asciidoc @@ -33,7 +33,8 @@ the stability of the cluster. Those queries can be categorised as follows: * Queries that need to do linear scans to identify matches: ** <> -** queries on <>, <>, <>, <> or <> fields +** queries on <>, <>, <>, <>, + <> or <> fields that are not indexed but have <> enabled * Queries that have a high up-front cost: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml index 8b9cde1ad6bea..663c099ed6fda 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml @@ -95,6 +95,9 @@ setup: non_indexed_ip: type: ip index: false + non_indexed_geo_point: + type: geo_point + index: false geo: type: keyword object: @@ -270,6 +273,19 @@ setup: - match: {fields.non_indexed_ip.ip.searchable: true} +--- +"Field caps for geo_point field with only doc values": + - skip: + version: " - 8.1.99" + reason: "doc values search was added in 8.2.0" + - do: + field_caps: + index: 'test1,test2,test3' + fields: non_indexed_geo_point + + - match: {fields.non_indexed_geo_point.geo_point.searchable: true} + + --- "Get object and nested field caps": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml index a0552b7f079fe..1657e78cf86ea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml @@ -45,6 +45,9 @@ setup: ip: type: ip index: false + geo_point: + type: geo_point + index: false - do: index: @@ -62,6 +65,7 @@ setup: keyword: "key1" boolean: "false" ip: "192.168.0.1" + geo_point: [13.5, 34.89] - do: index: @@ -79,6 +83,7 @@ setup: keyword: "key2" boolean: "true" ip: "192.168.0.2" + geo_point : [-63.24, 31.0] - do: indices.refresh: {} @@ -238,6 +243,9 @@ setup: --- "Test distance_feature query on date field where only doc values are enabled": + - skip: + version: " - 8.1.99" + reason: "doc values search was added in 8.2.0" - do: search: @@ -334,3 +342,42 @@ setup: index: test body: { query: { range: { ip: { gte: "192.168.0.1" } } } } - length: { hits.hits: 2 } + +--- +"Test geo shape query on geo_point field where only doc values are enabled": + - skip: + version: " - 8.1.99" + reason: "doc values search was added in 8.2.0" + + - do: + search: + index: test + body: + query: + geo_shape: + geo_point: + shape: + type: "envelope" + coordinates: [ [ -70, 32 ], [ -50, 30 ] ] + - match: {hits.total.value: 1} + +--- +"Test distance_feature query on geo_point field where only doc values are enabled": + - skip: + version: " - 8.1.99" + reason: "doc values search was added in 8.2.0" + + - do: + search: + index: test + body: + query: + bool: + should: + distance_feature: + field: geo_point + pivot: "1000km" + origin: [0.0, 0.0] + - length: { hits.hits: 2 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.1._id: "2" } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 25544471d7ac3..5f01f93375147 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -609,8 +609,13 @@ public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionCo if (isIndexed()) { return LongPoint.newDistanceFeatureQuery(name(), 1.0f, originLong, pivotLong); } else { - return new LongScriptFieldDistanceFeatureQuery(new Script(""), - ctx -> new SortedNumericDocValuesLongFieldScript(name(), context.lookup(), ctx), name(), originLong, pivotLong); + return new LongScriptFieldDistanceFeatureQuery( + new Script(""), + ctx -> new SortedNumericDocValuesLongFieldScript(name(), context.lookup(), ctx), + name(), + originLong, + pivotLong + ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index 520397f5b2578..40ebc9fa9e59d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -19,7 +19,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.fielddata.DateScriptFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.query.SearchExecutionContext; @@ -161,7 +160,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier lookup) { + public DateScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier lookup) { return new DateScriptFieldData.Builder(name(), leafFactory(lookup.get()), Resolution.MILLISECONDS.getDefaultToScriptField()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 2a753a6bd6465..ee2e33c3ed3a8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -37,9 +37,11 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.field.GeoPointDocValuesField; +import org.elasticsearch.script.field.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.support.MapXContentParser; @@ -268,6 +270,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public boolean isSearchable() { + return isIndexed() || hasDocValues(); + } + @Override protected Function, List> getFormatter(String format) { return GEO_FORMATTER_FACTORY.getFormatter(format, p -> new Point(p.getLon(), p.getLat())); @@ -284,6 +291,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relation, SearchExecutionContext context) { + failIfNotIndexedNorDocValuesFallback(context); final LatLonGeometry[] luceneGeometries = GeoShapeUtils.toLuceneGeometry(fieldName, context, shape, relation); if (luceneGeometries.length == 0) { return new MatchNoDocsQuery(); @@ -296,10 +304,15 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat } else { luceneRelation = relation.getLuceneRelation(); } - Query query = LatLonPoint.newGeometryQuery(fieldName, luceneRelation, luceneGeometries); - if (hasDocValues()) { - Query dvQuery = LatLonDocValuesField.newSlowGeometryQuery(fieldName, luceneRelation, luceneGeometries); - query = new IndexOrDocValuesQuery(query, dvQuery); + Query query; + if (isIndexed()) { + query = LatLonPoint.newGeometryQuery(fieldName, luceneRelation, luceneGeometries); + if (hasDocValues()) { + Query dvQuery = LatLonDocValuesField.newSlowGeometryQuery(fieldName, luceneRelation, luceneGeometries); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + } else { + query = LatLonDocValuesField.newSlowGeometryQuery(fieldName, luceneRelation, luceneGeometries); } return query; } @@ -312,6 +325,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S @Override public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) { + failIfNotIndexedNorDocValuesFallback(context); GeoPoint originGeoPoint; if (origin instanceof GeoPoint) { originGeoPoint = (GeoPoint) origin; @@ -326,8 +340,19 @@ public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionCo ); } double pivotDouble = DistanceUnit.DEFAULT.parse(pivot, DistanceUnit.DEFAULT); - // As we already apply boost in AbstractQueryBuilder::toQuery, we always passing a boost of 1.0 to distanceFeatureQuery - return LatLonPoint.newDistanceFeatureQuery(name(), 1.0f, originGeoPoint.lat(), originGeoPoint.lon(), pivotDouble); + if (isIndexed()) { + // As we already apply boost in AbstractQueryBuilder::toQuery, we always passing a boost of 1.0 to distanceFeatureQuery + return LatLonPoint.newDistanceFeatureQuery(name(), 1.0f, originGeoPoint.lat(), originGeoPoint.lon(), pivotDouble); + } else { + return new GeoPointScriptFieldDistanceFeatureQuery( + new Script(""), + ctx -> new SortedNumericDocValuesLongFieldScript(name(), context.lookup(), ctx), + name(), + originGeoPoint.lat(), + originGeoPoint.lon(), + pivotDouble + ); + } } } From 9cb5eddf12cb4047dc3ef7fb1bf04ac2f3b6707f Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 2 Feb 2022 09:01:03 +0100 Subject: [PATCH 4/8] Update docs/changelog/83395.yaml --- docs/changelog/83395.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/83395.yaml diff --git a/docs/changelog/83395.yaml b/docs/changelog/83395.yaml new file mode 100644 index 0000000000000..87d016df38f9e --- /dev/null +++ b/docs/changelog/83395.yaml @@ -0,0 +1,5 @@ +pr: 83395 +summary: Date script doc values +area: Mapping +type: enhancement +issues: [] From 94500d984ea9fabec295ae6f9a492c1659af41ad Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 2 Feb 2022 09:19:55 +0100 Subject: [PATCH 5/8] fix test --- .../elasticsearch/index/mapper/GeoPointFieldMapperTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 6750eda9dc361..922e20b410873 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -198,7 +198,7 @@ public void testIndexParameter() throws Exception { Mapper fieldMapper = mapper.mappers().getMapper("field"); assertThat(fieldMapper, instanceOf(GeoPointFieldMapper.class)); assertThat(((GeoPointFieldMapper) fieldMapper).fieldType().isIndexed(), equalTo(false)); - assertThat(((GeoPointFieldMapper) fieldMapper).fieldType().isSearchable(), equalTo(false)); + assertThat(((GeoPointFieldMapper) fieldMapper).fieldType().isSearchable(), equalTo(true)); } public void testMultiField() throws Exception { From e36e89792c4a889b7290dfea4dd46bef491512ff Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 2 Feb 2022 10:38:29 +0100 Subject: [PATCH 6/8] add comment --- .../script/field/SortedNumericDocValuesLongFieldScript.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java b/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java index 00b7af4215e2f..4b911c7d52a77 100644 --- a/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java @@ -31,6 +31,7 @@ public SortedNumericDocValuesLongFieldScript(String fieldName, SearchLookup look @Override protected void emitFromObject(Object v) { + // we only use doc-values, not _source, so no need to implement this method throw new UnsupportedOperationException(); } From 40badecfab3e6b99a2f89881fe3f4ec2d71ef5d4 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 2 Feb 2022 10:39:49 +0100 Subject: [PATCH 7/8] fix title --- docs/changelog/83395.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/83395.yaml b/docs/changelog/83395.yaml index 87d016df38f9e..6a7e0849e5775 100644 --- a/docs/changelog/83395.yaml +++ b/docs/changelog/83395.yaml @@ -1,5 +1,5 @@ pr: 83395 -summary: Date script doc values +summary: Allow doc-values only search on geo_point fields area: Mapping type: enhancement issues: [] From e2c9d3c1a84372dd27e8b9def354a3afa540dca4 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 2 Feb 2022 10:42:07 +0100 Subject: [PATCH 8/8] let's get this into 8.1.0 --- .../rest-api-spec/test/field_caps/10_basic.yml | 4 ++-- .../test/search/390_doc_values_search.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml index 663c099ed6fda..e1817c9c607ef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml @@ -276,8 +276,8 @@ setup: --- "Field caps for geo_point field with only doc values": - skip: - version: " - 8.1.99" - reason: "doc values search was added in 8.2.0" + version: " - 8.0.99" + reason: "doc values search was added in 8.1.0" - do: field_caps: index: 'test1,test2,test3' diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml index 1657e78cf86ea..f497acfce5626 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml @@ -244,8 +244,8 @@ setup: --- "Test distance_feature query on date field where only doc values are enabled": - skip: - version: " - 8.1.99" - reason: "doc values search was added in 8.2.0" + version: " - 8.0.99" + reason: "doc values search was added in 8.1.0" - do: search: @@ -346,8 +346,8 @@ setup: --- "Test geo shape query on geo_point field where only doc values are enabled": - skip: - version: " - 8.1.99" - reason: "doc values search was added in 8.2.0" + version: " - 8.0.99" + reason: "doc values search was added in 8.1.0" - do: search: @@ -364,8 +364,8 @@ setup: --- "Test distance_feature query on geo_point field where only doc values are enabled": - skip: - version: " - 8.1.99" - reason: "doc values search was added in 8.2.0" + version: " - 8.0.99" + reason: "doc values search was added in 8.1.0" - do: search: