diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc index 122d555036cea..1dc585b3975fd 100644 --- a/docs/reference/mapping/params/doc-values.asciidoc +++ b/docs/reference/mapping/params/doc-values.asciidoc @@ -17,8 +17,8 @@ makes this data access pattern possible. They store the same values as the sorting and aggregations. Doc values are supported on almost all field types, with the __notable exception of `text` and `annotated_text` fields__. -<>, <>, the <> -and the <> +<>, <>, the <>, +the <> and the <> can also be queried using term or range-based queries when they are not <> but only have doc values enabled. Query performance on doc values is much slower than on index structures, but diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc index b81c63da98c10..2e598e40bbacc 100644 --- a/docs/reference/mapping/types/ip.asciidoc +++ b/docs/reference/mapping/types/ip.asciidoc @@ -57,7 +57,10 @@ The following parameters are accepted by `ip` fields: <>:: - Should the field be searchable? Accepts `true` (default) and `false`. + Should the field be quickly searchable? Accepts `true` (default) and + `false`. Fields that only have <> + enabled can still be queried using term or range-based queries, + albeit slower. <>:: diff --git a/docs/reference/query-dsl.asciidoc b/docs/reference/query-dsl.asciidoc index c0fa107ab1468..fa9334f390ed1 100644 --- a/docs/reference/query-dsl.asciidoc +++ b/docs/reference/query-dsl.asciidoc @@ -33,8 +33,8 @@ the stability of the cluster. Those queries can be categorised as follows: * Queries that need to do linear scans to identify matches: ** <> -** queries on <>, <>, <>, or <> fields that are not indexed - but have <> enabled +** queries on <>, <>, <>, <> or <> fields + that are not indexed but have <> enabled * Queries that have a high up-front cost: ** <> (except on diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml index 06067c6f4c62d..8b9cde1ad6bea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/10_basic.yml @@ -92,6 +92,9 @@ setup: non_indexed_boolean: type: boolean index: false + non_indexed_ip: + type: ip + index: false geo: type: keyword object: @@ -255,6 +258,18 @@ setup: - match: {fields.non_indexed_boolean.boolean.searchable: true} +--- +"Field caps for ip field with only doc values": + - skip: + version: " - 8.0.99" + reason: "doc values search was added in 8.1.0" + - do: + field_caps: + index: 'test1,test2,test3' + fields: non_indexed_ip + + - match: {fields.non_indexed_ip.ip.searchable: true} + --- "Get object and nested field caps": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml index f39f89c876485..323c521f4d128 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/390_doc_values_search.yml @@ -42,6 +42,9 @@ setup: boolean: type: boolean index: false + ip: + type: ip + index: false - do: index: @@ -58,6 +61,7 @@ setup: date: "2017/01/01" keyword: "key1" boolean: "false" + ip: "192.168.0.1" - do: index: @@ -74,6 +78,7 @@ setup: date: "2017/01/02" keyword: "key2" boolean: "true" + ip: "192.168.0.2" - do: indices.refresh: {} @@ -284,3 +289,30 @@ setup: index: test body: { query: { range: { boolean: { gte: "false" } } } } - length: { hits.hits: 2 } + +--- +"Test match query on ip field where only doc values are enabled": + + - do: + search: + index: test + body: { query: { match: { ip: { query: "192.168.0.1" } } } } + - length: { hits.hits: 1 } + +--- +"Test terms query on ip field where only doc values are enabled": + + - do: + search: + index: test + body: { query: { terms: { ip: [ "192.168.0.1", "192.168.0.2" ] } } } + - length: { hits.hits: 2 } + +--- +"Test range query on ip field where only doc values are enabled": + + - do: + search: + index: test + body: { query: { range: { ip: { gte: "192.168.0.1" } } } } + - length: { hits.hits: 2 } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index cdd1c53b97e03..9efbbe27ec8dd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -14,6 +14,7 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; @@ -204,7 +205,15 @@ public IpFieldType( } public IpFieldType(String name) { - this(name, true, false, true, null, null, Collections.emptyMap(), false); + this(name, true, true); + } + + public IpFieldType(String name, boolean isIndexed) { + this(name, isIndexed, true); + } + + public IpFieldType(String name, boolean isIndexed, boolean hasDocValues) { + this(name, isIndexed, false, hasDocValues, null, null, Collections.emptyMap(), false); } @Override @@ -212,6 +221,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public boolean isSearchable() { + return isIndexed() || hasDocValues(); + } + @Override public boolean mayExistInIndex(SearchExecutionContext context) { return context.fieldExistsInIndex(name()); @@ -252,9 +266,10 @@ protected Object parseSourceValue(Object value) { @Override public Query termQuery(Object value, @Nullable SearchExecutionContext context) { - failIfNotIndexed(); + failIfNotIndexedNorDocValuesFallback(context); + Query query; if (value instanceof InetAddress) { - return InetAddressPoint.newExactQuery(name(), (InetAddress) value); + query = InetAddressPoint.newExactQuery(name(), (InetAddress) value); } else { if (value instanceof BytesRef) { value = ((BytesRef) value).utf8ToString(); @@ -262,15 +277,37 @@ public Query termQuery(Object value, @Nullable SearchExecutionContext context) { String term = value.toString(); if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); - return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + } else { + InetAddress address = InetAddresses.forString(term); + query = InetAddressPoint.newExactQuery(name(), address); } - InetAddress address = InetAddresses.forString(term); - return InetAddressPoint.newExactQuery(name(), address); + } + if (isIndexed()) { + return query; + } else { + return convertToDocValuesQuery(query); } } + static Query convertToDocValuesQuery(Query query) { + assert query instanceof PointRangeQuery; + PointRangeQuery pointRangeQuery = (PointRangeQuery) query; + return SortedSetDocValuesField.newSlowRangeQuery( + pointRangeQuery.getField(), + new BytesRef(pointRangeQuery.getLowerPoint()), + new BytesRef(pointRangeQuery.getUpperPoint()), + true, + true + ); + } + @Override public Query termsQuery(Collection values, SearchExecutionContext context) { + failIfNotIndexedNorDocValuesFallback(context); + if (isIndexed() == false) { + return super.termsQuery(values, context); + } InetAddress[] addresses = new InetAddress[values.size()]; int i = 0; for (Object value : values) { @@ -301,14 +338,15 @@ public Query rangeQuery( boolean includeUpper, SearchExecutionContext context ) { - failIfNotIndexed(); - return rangeQuery( - lowerTerm, - upperTerm, - includeLower, - includeUpper, - (lower, upper) -> InetAddressPoint.newRangeQuery(name(), lower, upper) - ); + failIfNotIndexedNorDocValuesFallback(context); + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> { + Query query = InetAddressPoint.newRangeQuery(name(), lower, upper); + if (isIndexed()) { + return query; + } else { + return convertToDocValuesQuery(query); + } + }); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java index 8e7c678f95857..ec0348d23146f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.List; +import static org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.convertToDocValuesQuery; + public class IpFieldTypeTests extends FieldTypeTestCase { public void testValueFormat() throws Exception { @@ -51,31 +53,59 @@ public void testTermQuery() { MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); String ip = "2001:db8::2:1"; - assertEquals(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)), ft.termQuery(ip, null)); + assertEquals(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)), ft.termQuery(ip, MOCK_CONTEXT)); ip = "192.168.1.7"; - assertEquals(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)), ft.termQuery(ip, null)); + assertEquals(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)), ft.termQuery(ip, MOCK_CONTEXT)); ip = "2001:db8::2:1"; String prefix = ip + "/64"; - assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null)); + assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, MOCK_CONTEXT)); + + ip = "192.168.1.7"; + prefix = ip + "/16"; + assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, MOCK_CONTEXT)); + + ft = new IpFieldMapper.IpFieldType("field", false); + + ip = "2001:db8::2:1"; + assertEquals( + convertToDocValuesQuery(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip))), + ft.termQuery(ip, MOCK_CONTEXT) + ); + + ip = "192.168.1.7"; + assertEquals( + convertToDocValuesQuery(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip))), + ft.termQuery(ip, MOCK_CONTEXT) + ); + + ip = "2001:db8::2:1"; + prefix = ip + "/64"; + assertEquals( + convertToDocValuesQuery(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64)), + ft.termQuery(prefix, MOCK_CONTEXT) + ); ip = "192.168.1.7"; prefix = ip + "/16"; - assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null)); + assertEquals( + convertToDocValuesQuery(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16)), + ft.termQuery(prefix, MOCK_CONTEXT) + ); MappedFieldType unsearchable = new IpFieldMapper.IpFieldType( "field", false, false, - true, + false, null, null, Collections.emptyMap(), false ); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", null)); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", MOCK_CONTEXT)); + assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); } public void testTermsQuery() { @@ -83,21 +113,21 @@ public void testTermsQuery() { assertEquals( InetAddressPoint.newSetQuery("field", InetAddresses.forString("::2"), InetAddresses.forString("::5")), - ft.termsQuery(Arrays.asList(InetAddresses.forString("::2"), InetAddresses.forString("::5")), null) + ft.termsQuery(Arrays.asList(InetAddresses.forString("::2"), InetAddresses.forString("::5")), MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newSetQuery("field", InetAddresses.forString("::2"), InetAddresses.forString("::5")), - ft.termsQuery(Arrays.asList("::2", "::5"), null) + ft.termsQuery(Arrays.asList("::2", "::5"), MOCK_CONTEXT) ); // if the list includes a prefix query we fallback to a bool query assertEquals( new ConstantScoreQuery( - new BooleanQuery.Builder().add(ft.termQuery("::42", null), Occur.SHOULD) + new BooleanQuery.Builder().add(ft.termQuery("::42", MOCK_CONTEXT), Occur.SHOULD) .add(ft.termQuery("::2/16", null), Occur.SHOULD) .build() ), - ft.termsQuery(Arrays.asList("::42", "::2/16"), null) + ft.termsQuery(Arrays.asList("::42", "::2/16"), MOCK_CONTEXT) ); } @@ -106,47 +136,47 @@ public void testRangeQuery() { assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddressPoint.MAX_VALUE), - ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, null) + ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.2.0")), - ft.rangeQuery(null, "192.168.2.0", randomBoolean(), true, null, null, null, null) + ft.rangeQuery(null, "192.168.2.0", randomBoolean(), true, null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.1.255")), - ft.rangeQuery(null, "192.168.2.0", randomBoolean(), false, null, null, null, null) + ft.rangeQuery(null, "192.168.2.0", randomBoolean(), false, null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddressPoint.MAX_VALUE), - ft.rangeQuery("2001:db8::", null, true, randomBoolean(), null, null, null, null) + ft.rangeQuery("2001:db8::", null, true, randomBoolean(), null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddressPoint.MAX_VALUE), - ft.rangeQuery("2001:db8::", null, false, randomBoolean(), null, null, null, null) + ft.rangeQuery("2001:db8::", null, false, randomBoolean(), null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddresses.forString("2001:db8::ffff")), - ft.rangeQuery("2001:db8::", "2001:db8::ffff", true, true, null, null, null, null) + ft.rangeQuery("2001:db8::", "2001:db8::ffff", true, true, null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddresses.forString("2001:db8::fffe")), - ft.rangeQuery("2001:db8::", "2001:db8::ffff", false, false, null, null, null, null) + ft.rangeQuery("2001:db8::", "2001:db8::ffff", false, false, null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::2"), InetAddresses.forString("2001:db8::")), // same lo/hi values but inclusive=false so this won't match anything - ft.rangeQuery("2001:db8::1", "2001:db8::1", false, false, null, null, null, null) + ft.rangeQuery("2001:db8::1", "2001:db8::1", false, false, null, null, null, MOCK_CONTEXT) ); // Upper bound is the min IP and is not inclusive - assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("::", "::", true, false, null, null, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("::", "::", true, false, null, null, null, MOCK_CONTEXT)); // Lower bound is the max IP and is not inclusive assertEquals( @@ -159,33 +189,132 @@ public void testRangeQuery() { null, null, null, - null + MOCK_CONTEXT ) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("::fffe:ffff:ffff")), // same lo/hi values but inclusive=false so this won't match anything - ft.rangeQuery("::", "0.0.0.0", true, false, null, null, null, null) + ft.rangeQuery("::", "0.0.0.0", true, false, null, null, null, MOCK_CONTEXT) ); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::1:0:0:0"), InetAddressPoint.MAX_VALUE), // same lo/hi values but inclusive=false so this won't match anything - ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true, null, null, null, null) + ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true, null, null, null, MOCK_CONTEXT) ); assertEquals( // lower bound is ipv4, upper bound is ipv6 InetAddressPoint.newRangeQuery("field", InetAddresses.forString("192.168.1.7"), InetAddresses.forString("2001:db8::")), - ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true, null, null, null, null) + ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true, null, null, null, MOCK_CONTEXT) + ); + + ft = new IpFieldMapper.IpFieldType("field", false); + + assertEquals( + convertToDocValuesQuery(InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddressPoint.MAX_VALUE)), + ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.2.0")) + ), + ft.rangeQuery(null, "192.168.2.0", randomBoolean(), true, null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.1.255")) + ), + ft.rangeQuery(null, "192.168.2.0", randomBoolean(), false, null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddressPoint.MAX_VALUE) + ), + ft.rangeQuery("2001:db8::", null, true, randomBoolean(), null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddressPoint.MAX_VALUE) + ), + ft.rangeQuery("2001:db8::", null, false, randomBoolean(), null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddresses.forString("2001:db8::ffff")) + ), + ft.rangeQuery("2001:db8::", "2001:db8::ffff", true, true, null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddresses.forString("2001:db8::fffe")) + ), + ft.rangeQuery("2001:db8::", "2001:db8::ffff", false, false, null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::2"), InetAddresses.forString("2001:db8::")) + ), + // same lo/hi values but inclusive=false so this won't match anything + ft.rangeQuery("2001:db8::1", "2001:db8::1", false, false, null, null, null, MOCK_CONTEXT) + ); + + // Upper bound is the min IP and is not inclusive + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("::", "::", true, false, null, null, null, MOCK_CONTEXT)); + + // Lower bound is the max IP and is not inclusive + assertEquals( + new MatchNoDocsQuery(), + ft.rangeQuery( + "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + false, + true, + null, + null, + null, + MOCK_CONTEXT + ) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("::fffe:ffff:ffff")) + ), + // same lo/hi values but inclusive=false so this won't match anything + ft.rangeQuery("::", "0.0.0.0", true, false, null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::1:0:0:0"), InetAddressPoint.MAX_VALUE) + ), + // same lo/hi values but inclusive=false so this won't match anything + ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true, null, null, null, MOCK_CONTEXT) + ); + + assertEquals( + // lower bound is ipv4, upper bound is ipv6 + convertToDocValuesQuery( + InetAddressPoint.newRangeQuery("field", InetAddresses.forString("192.168.1.7"), InetAddresses.forString("2001:db8::")) + ), + ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true, null, null, null, MOCK_CONTEXT) ); MappedFieldType unsearchable = new IpFieldMapper.IpFieldType( "field", false, false, - true, + false, null, null, Collections.emptyMap(), @@ -193,9 +322,9 @@ public void testRangeQuery() { ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unsearchable.rangeQuery("::1", "2001::", true, true, null, null, null, null) + () -> unsearchable.rangeQuery("::1", "2001::", true, true, null, null, null, MOCK_CONTEXT) ); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); } public void testFetchSourceValue() throws IOException {