From ff6f44f0c9a1bfc4c6dca92dc10da82d8974871d Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 17 Jan 2022 14:11:34 +0100 Subject: [PATCH 1/8] Auto-convert old mappings --- .../cluster/metadata/IndexMetadata.java | 62 +- .../index/mapper/LegacyMappingConverter.java | 223 + .../snapshots/RestoreService.java | 47 + .../mapper/LegacyMappingConverterTests.java | 48 + .../elasticsearch/index/mapper/filebeat5.json | 683 +++ .../elasticsearch/index/mapper/filebeat6.json | 4903 +++++++++++++++++ .../oldrepos/OldRepositoryAccessIT.java | 38 + 7 files changed, 6000 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java create mode 100644 server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json create mode 100644 server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 10ff36a5b855f..32850585b3a59 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -39,6 +39,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.LegacyMappingConverter; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexLongFieldRange; @@ -60,6 +61,7 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -1899,8 +1901,19 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE } builder.settings(settings); } else if ("mappings".equals(currentFieldName)) { - // don't try to parse these for now - parser.skipChildren(); + MapBuilder mappingSourceBuilder = MapBuilder.newMapBuilder(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + String mappingType = currentFieldName; + mappingSourceBuilder.put(mappingType, parser.mapOrdered()); + } else { + throw new IllegalArgumentException("Unexpected token: " + token); + } + } + Map mapping = mappingSourceBuilder.map(); + handleLegacyMapping(builder, mapping); } else if ("in_sync_allocations".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -1925,7 +1938,16 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE } else if (token == XContentParser.Token.START_ARRAY) { if ("mappings".equals(currentFieldName)) { // don't try to parse these for now - parser.skipChildren(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + Map mapping; + if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + CompressedXContent compressedXContent = new CompressedXContent(parser.binaryValue()); + mapping = XContentHelper.convertToMap(compressedXContent.compressedReference(), true).v2(); + } else { + mapping = parser.mapOrdered(); + } + handleLegacyMapping(builder, mapping); + } } else { parser.skipChildren(); } @@ -1949,12 +1971,44 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE } XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - builder.putMapping(MappingMetadata.EMPTY_MAPPINGS); // just make sure it's not empty so that _source can be read + if (builder.mapping() == null) { + builder.putMapping(MappingMetadata.EMPTY_MAPPINGS); // just make sure it's not empty so that _source can be read + } IndexMetadata indexMetadata = builder.build(); assert indexMetadata.getCreationVersion().before(Version.CURRENT.minimumIndexCompatibilityVersion()); return indexMetadata; } + + private static void handleLegacyMapping(Builder builder, Map mapping) { + //Map adaptedMapping = new + + // we don't need to obey any routing here stuff is read-only anyway and get is disabled + // final String mapping = "{ \"_doc\" : { \"enabled\": false, \"_meta\": " + mmd.source().string() + " } }"; + +// LegacyMappingConverter legacyMappingConverter = new LegacyMappingConverter(); +// @SuppressWarnings("unchecked") +// Map newMapping = (Map) legacyMappingConverter.extractNewMapping(mapping, true); +// mapping = newMapping; + + if (mapping.size() == 1) { + String mappingType = mapping.keySet().iterator().next(); +// @SuppressWarnings("unchecked") +// Map typelessMapping = (Map) mapping.get(mappingType); +// Map runtimeFieldMapping = legacyMappingConverter.extractRuntimeFieldMapping(typelessMapping); +// if (runtimeFieldMapping.isEmpty() == false) { +// // TODO: only add those runtime fields that don't already exist +// LinkedHashMap newTypelessMapping = new LinkedHashMap<>(); +// newTypelessMapping.put("runtime", runtimeFieldMapping); +// newTypelessMapping.putAll(typelessMapping); +// mapping.put(mappingType, newTypelessMapping); +// } + builder.putMapping(new MappingMetadata(mappingType, mapping)); + } else if (mapping.size() > 1) { + // TODO: handle this better + throw new RuntimeException("can't handle multiple types"); + } + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java new file mode 100644 index 0000000000000..ee4a138a86e6b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.action.admin.cluster.stats.MappingVisitor; +import org.elasticsearch.common.xcontent.support.XContentMapValues; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class LegacyMappingConverter { + + public Map extractRuntimeFieldMapping(Map mapping) { + final Map result = new LinkedHashMap<>(); + MappingVisitor.visitMapping(mapping, (key, map) -> { + Object type = map.get("type"); + if (type instanceof String) { + String typeAsString = (String) type; + // copy over known basic types for which there are runtime fields + // first auto-convert some types + switch (typeAsString) { + case "byte": + case "short": + case "integer": + typeAsString = "long"; + break; + case "half_float": + case "float": + typeAsString = "double"; + break; + default: + // ignore + break; + } + switch (typeAsString) { + case "boolean": + case "date": + case "double": + case "geo_point": + case "ip": + case "keyword": + case "long": + // copy over only format parameter if it exists + // TODO: converting multi-fields to default runtime fields does not make sense + // (as field name in mapping is not equal to field name in _source) + // This means we need to add a script that returns the correct field from _source + final Map runtimeFieldDefinition = new LinkedHashMap<>(); + runtimeFieldDefinition.put("type", typeAsString); + Object format = map.get("format"); + if (format instanceof String) { + runtimeFieldDefinition.put("format", format); + } + result.put(key, runtimeFieldDefinition); + break; + default: + // ignore + break; + } + } + }); + return result; + } + + public Map extractDocValuesMapping(Map mapping) { + final Map result = new LinkedHashMap<>(); + MappingVisitor.visitMapping(mapping, (key, map) -> { + Object type = map.get("type"); + if (type instanceof String) { + // copy over known basic types for which there are doc-value only implementations + switch ((String) type) { + case "date": + case "date_nanos": + case "long": + case "integer": + case "short": + case "byte": + case "double": + case "float": + case "half_float": + case "scaled_float": + final Object docValues = map.get("doc_values"); + // check that doc_values had not been disabled + if (docValues != null && XContentMapValues.nodeBooleanValue(docValues) == false) { + return; + } + final Map docValueOnlyDefinition = new LinkedHashMap<>(map); + docValueOnlyDefinition.put("index", false); + // TODO: this does not work, as the key is "dotted" + // we would need to recreate hierarchical mapping + result.put(key, docValueOnlyDefinition); + break; + default: + // ignore + } + } + }); + return result; + } + + public Map extractNewMapping(Map mapping, boolean withTypes) { + final Map result = new LinkedHashMap<>(); + if (withTypes) { + for (Map.Entry entry : mapping.entrySet()) { + @SuppressWarnings("unchecked") + Map actualMapping = (Map) entry.getValue(); + result.put(entry.getKey(), extractNewMapping(actualMapping, false)); + } + return result; + } + + Object properties = mapping.get("properties"); + if (properties instanceof Map) { + @SuppressWarnings("unchecked") + Map propertiesAsMap = (Map) properties; + Map newPropertiesMap = new LinkedHashMap<>(); + for (Map.Entry entry : propertiesAsMap.entrySet()) { + final Object v = entry.getValue(); + if (v instanceof Map) { + @SuppressWarnings("unchecked") + Map fieldMapping = (Map) v; + Object type = fieldMapping.get("type"); + Map fieldMap = new LinkedHashMap<>(); + if (type instanceof String) { + // copy over known types + switch ((String) type) { + case "binary": + case "boolean": + case "keyword": + case "long": + case "integer": + case "short": + case "byte": + case "double": + case "float": + case "half_float": + case "scaled_float": + case "unsigned_long": + case "date": + case "date_nanos": + case "alias": + case "object": + case "flattened": + case "nested": + case "join": + case "integer_range": + case "float_range": + case "long_range": + case "double_range": + case "date_range": + case "ip_range": + case "ip": + case "version": + case "murmur3": + case "aggregate_metric_double": + case "histogram": + case "text": + case "match_only_text": + case "annotated-text": + case "completion": + case "search_as_you_type": + case "token_count": + case "dense_vector": + case "sparse_vector": + case "rank_feature": + case "rank_features": + case "geo_point": + case "geo_shape": + case "point": + case "shape": + case "percolator": + fieldMap.putAll(fieldMapping); + break; + default: + assert false; + // ignore + } + } + + // Multi fields + Object fieldsO = fieldMapping.get("fields"); + if (fieldsO instanceof Map) { + @SuppressWarnings("unchecked") + Map fields = (Map) fieldsO; + Map fieldsMap = new LinkedHashMap<>(); + for (Map.Entry subfieldEntry : fields.entrySet()) { + Object v2 = subfieldEntry.getValue(); + if (v2 instanceof Map) { + @SuppressWarnings("unchecked") + Map fieldMapping2 = (Map) v2; + String key = entry.getKey() + "." + subfieldEntry.getKey(); + Map extracted = extractNewMapping(fieldMapping2, false); + if (extracted.isEmpty() == false) { + fieldsMap.put(key, extracted); + } + } + } + if (fieldsMap.isEmpty() == false) { + fieldMap.put("fields", fieldsMap); + } + } + + Map extracted = extractNewMapping(fieldMapping, false); + fieldMap.putAll(extracted); + + if (fieldMap.isEmpty() == false) { + newPropertiesMap.put(entry.getKey(), fieldMap); + } + } + } + if (newPropertiesMap.isEmpty() == false) { + result.put("properties", newPropertiesMap); + } + } + + return result; + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 3950597155f82..cae3c0e71139d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; @@ -78,6 +79,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -1294,6 +1296,11 @@ public ClusterState execute(ClusterState currentState) { request.indexSettings(), request.ignoreIndexSettings() ); + if (snapshotIndexMetadata.getCreationVersion().before( + currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion())) { + // adapt mappings so that they can be read + snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata); + } try { snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion); } catch (Exception ex) { @@ -1582,6 +1589,46 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) } } + private IndexMetadata convertLegacyIndex(IndexMetadata snapshotIndexMetadata) { + MappingMetadata mappingMetadata = snapshotIndexMetadata.mapping(); + Map loadedMappingSource = mappingMetadata.sourceAsMap(); + + // store old mapping under _meta/legacy-mapping + Map legacyMapping = new LinkedHashMap<>(); + boolean sourceOnlySnapshot = snapshotIndexMetadata.getSettings().getAsBoolean("index.source_only", false); + if (sourceOnlySnapshot) { + // actual mapping is under "_meta", and keyed by _type + Object sourceOnlyMeta = loadedMappingSource.get("_meta"); + if (sourceOnlyMeta instanceof Map sourceOnlyMetaMap) { + if (sourceOnlyMetaMap.size() == 1) { + if (sourceOnlyMetaMap.values().iterator().next() instanceof Map sourceOnlyLegacyMapping) { + legacyMapping.put("legacy-mapping", sourceOnlyLegacyMapping); + } + } + } + } else { + legacyMapping.put("legacy-mapping", loadedMappingSource); + } + + Map newMappingSource = new LinkedHashMap<>(); + newMappingSource.put("_meta", legacyMapping); + + // copy over _meta fields of original index into proper place? + // preserve legacy runtime fields + Object legacyRuntimeFields = loadedMappingSource.get("runtime"); + if (legacyRuntimeFields instanceof Map) { + @SuppressWarnings("unchecked") + Map legRuntimeFields = (Map) legacyRuntimeFields; + newMappingSource.put("runtime", legRuntimeFields); + } + Map newMapping = new LinkedHashMap<>(); + newMapping.put(mappingMetadata.type(), newMappingSource); + // TODO: _routing? Perhaps we don't need to obey any routing here as stuff is read-only anyway and get API will be disabled + return IndexMetadata.builder(snapshotIndexMetadata) + .putMapping(new MappingMetadata(mappingMetadata.type(), newMapping)) + .build(); + } + private static IndexMetadata.Builder restoreToCreateNewIndex(IndexMetadata snapshotIndexMetadata, String renamedIndexName) { return IndexMetadata.builder(snapshotIndexMetadata) .state(IndexMetadata.State.OPEN) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java new file mode 100644 index 0000000000000..7338c1d327aa3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; + +public class LegacyMappingConverterTests extends ESTestCase { + + public void testFilebeat5() throws IOException { + LegacyMappingConverter converter = new LegacyMappingConverter(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, + copyToStringFromClasspath("/org/elasticsearch/index/mapper/filebeat5.json"))) { + Map mapping = parser.mapOrdered(); + Map newMapping = converter.extractNewMapping(mapping, true); + logger.info("New mapping: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(newMapping))); + Map runtimeFieldMapping = converter.extractRuntimeFieldMapping(newMapping); + logger.info("Runtime fields: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(runtimeFieldMapping))); + assertEquals(137, runtimeFieldMapping.keySet().size()); + } + } + + public void testFilebeat6() throws IOException { + LegacyMappingConverter converter = new LegacyMappingConverter(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, + copyToStringFromClasspath("/org/elasticsearch/index/mapper/filebeat6.json"))) { + Map mapping = parser.mapOrdered(); + Map newMapping = converter.extractNewMapping(mapping, true); + logger.info("New mapping: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(newMapping))); + Map runtimeFieldMapping = converter.extractRuntimeFieldMapping(newMapping); + logger.info("Runtime fields: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(runtimeFieldMapping))); + assertEquals(1160, runtimeFieldMapping.keySet().size()); + } + } +} diff --git a/server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json b/server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json new file mode 100644 index 0000000000000..c6de6a166b115 --- /dev/null +++ b/server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json @@ -0,0 +1,683 @@ +{ + "_default_": { + "_meta": { + "version": "5.6.17" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "apache2": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "auditd": { + "properties": { + "log": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "acct": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ppid": { + "ignore_above": 1024, + "type": "keyword" + }, + "record_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "res": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + } + } + } + } + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileset": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "input_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "offset": { + "type": "long" + }, + "read_timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "properties": { + "gid": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "event": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "useradd": { + "properties": { + "gid": { + "type": "long" + }, + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "type": "long" + } + } + } + } + }, + "syslog": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + } +} diff --git a/server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json b/server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json new file mode 100644 index 0000000000000..a5d14ac75a1f7 --- /dev/null +++ b/server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json @@ -0,0 +1,4903 @@ +{ + "doc": { + "_meta": { + "version": "6.8.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "kibana.log.meta": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "kibana.log.meta.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "apache2": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "ssl": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "index": false, + "norms": false, + "type": "text" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "auditd": { + "properties": { + "log": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "acct": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ppid": { + "ignore_above": 1024, + "type": "keyword" + }, + "record_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "res": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + } + } + } + } + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cloud": { + "properties": { + "availability_zone": { + "path": "meta.cloud.availability_zone", + "type": "alias" + }, + "instance": { + "properties": { + "id": { + "path": "meta.cloud.instance_id", + "type": "alias" + }, + "name": { + "path": "meta.cloud.instance_name", + "type": "alias" + } + } + }, + "machine": { + "properties": { + "type": { + "path": "meta.cloud.machine_type", + "type": "alias" + } + } + }, + "project": { + "properties": { + "id": { + "path": "meta.cloud.project_id", + "type": "alias" + } + } + }, + "provider": { + "path": "meta.cloud.provider", + "type": "alias" + }, + "region": { + "path": "meta.cloud.region", + "type": "alias" + } + } + }, + "container": { + "properties": { + "id": { + "path": "docker.container.id", + "type": "alias" + }, + "image": { + "properties": { + "name": { + "path": "docker.container.image", + "type": "alias" + } + } + }, + "name": { + "path": "docker.container.name", + "type": "alias" + } + } + }, + "destination": { + "properties": { + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "docker": { + "properties": { + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "elasticsearch": { + "properties": { + "audit": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "indices": { + "ignore_above": 1024, + "type": "keyword" + }, + "layer": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin_address": { + "type": "ip" + }, + "origin_port": { + "type": "long" + }, + "origin_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "principal": { + "ignore_above": 1024, + "type": "keyword" + }, + "realm": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_body": { + "norms": false, + "type": "text" + }, + "request_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_method": { + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + }, + "uri": { + "ignore_above": 1024, + "type": "keyword" + }, + "uri_params": { + "norms": false, + "type": "text" + }, + "user_realm": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "deprecation": { + "properties": {} + }, + "gc": { + "properties": { + "heap": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "jvm_runtime_sec": { + "type": "float" + }, + "old_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "phase": { + "properties": { + "class_unload_time_sec": { + "type": "float" + }, + "cpu_time": { + "properties": { + "real_sec": { + "type": "float" + }, + "sys_sec": { + "type": "float" + }, + "user_sec": { + "type": "float" + } + } + }, + "duration_sec": { + "type": "float" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "parallel_rescan_time_sec": { + "type": "float" + }, + "scrub_string_table_time_sec": { + "type": "float" + }, + "scrub_symbol_table_time_sec": { + "type": "float" + }, + "weak_refs_processing_time_sec": { + "type": "float" + } + } + }, + "stopping_threads_time_sec": { + "type": "float" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threads_total_stop_time_sec": { + "type": "float" + }, + "young_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + } + } + }, + "index": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "node": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "gc": { + "properties": { + "collection_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "observation_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "overhead_seq": { + "type": "long" + }, + "young": { + "properties": { + "one": { + "type": "long" + }, + "two": { + "type": "long" + } + } + } + } + } + } + }, + "shard": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "extra_source": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "routing": { + "ignore_above": 1024, + "type": "keyword" + }, + "search_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_query": { + "norms": false, + "type": "text" + }, + "stats": { + "norms": false, + "type": "text" + }, + "took": { + "norms": false, + "type": "text" + }, + "took_millis": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_hits": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_shards": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "types": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "error": { + "properties": { + "code": { + "type": "long" + }, + "message": { + "norms": false, + "type": "text" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "module": { + "path": "fileset.module", + "type": "alias" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "path": "beat.timezone", + "type": "alias" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + } + } + }, + "fileset": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "haproxy": { + "properties": { + "backend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "backend_queue": { + "type": "long" + }, + "bind_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes_read": { + "type": "long" + }, + "client": { + "properties": { + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + } + } + }, + "connection_wait_time_ms": { + "type": "long" + }, + "connections": { + "properties": { + "active": { + "type": "long" + }, + "backend": { + "type": "long" + }, + "frontend": { + "type": "long" + }, + "retries": { + "type": "long" + }, + "server": { + "type": "long" + } + } + }, + "destination": { + "properties": { + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + } + } + }, + "error_message": { + "norms": false, + "type": "text" + }, + "frontend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "norms": false, + "type": "text" + }, + "raw_request_line": { + "norms": false, + "type": "text" + }, + "time_active_ms": { + "type": "long" + }, + "time_wait_ms": { + "type": "long" + }, + "time_wait_without_data_ms": { + "type": "long" + } + } + }, + "response": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "norms": false, + "type": "text" + }, + "status_code": { + "type": "long" + } + } + } + } + }, + "mode": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "process_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_queue": { + "type": "long" + }, + "source": { + "norms": false, + "type": "text" + }, + "tcp": { + "properties": { + "connection_waiting_time_ms": { + "type": "long" + }, + "processing_time_ms": { + "type": "long" + } + } + }, + "termination_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_backend_connect": { + "type": "long" + }, + "time_queue": { + "type": "long" + }, + "total_waiting_time_ms": { + "type": "long" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "containerized": { + "type": "boolean" + }, + "hostname": { + "path": "beat.hostname", + "type": "alias" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "method": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "content_length": { + "type": "long" + }, + "elapsed_time": { + "type": "long" + }, + "status_code": { + "type": "long" + } + } + } + } + }, + "icinga": { + "properties": { + "debug": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "main": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "startup": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "iis": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_received": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query_string": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_time_ms": { + "type": "long" + }, + "response_code": { + "type": "long" + }, + "server_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "site_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sub_status": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "index": false, + "norms": false, + "type": "text" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "win32_status": { + "type": "long" + } + } + }, + "error": { + "properties": { + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "queue_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason_phrase": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_port": { + "type": "long" + }, + "response_code": { + "type": "long" + }, + "server_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_port": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "input": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "iptables": { + "properties": { + "ether_type": { + "type": "long" + }, + "flow_label": { + "type": "long" + }, + "fragment_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment_offset": { + "type": "long" + }, + "icmp": { + "properties": { + "code": { + "type": "long" + }, + "id": { + "type": "long" + }, + "parameter": { + "type": "long" + }, + "redirect": { + "type": "ip" + }, + "seq": { + "type": "long" + }, + "type": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "incomplete_bytes": { + "type": "long" + }, + "input_device": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "output_device": { + "ignore_above": 1024, + "type": "keyword" + }, + "precedence_bits": { + "type": "short" + }, + "tcp": { + "properties": { + "ack": { + "type": "long" + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "reserved_bits": { + "type": "short" + }, + "seq": { + "type": "long" + }, + "window": { + "type": "long" + } + } + }, + "tos": { + "type": "long" + }, + "ttl": { + "type": "long" + }, + "ubiquiti": { + "properties": { + "input_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "output_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_set": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "udp": { + "properties": { + "length": { + "type": "long" + } + } + } + } + }, + "kafka": { + "properties": { + "log": { + "properties": { + "class": { + "norms": false, + "type": "text" + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "trace": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "norms": false, + "type": "text" + }, + "message": { + "norms": false, + "type": "text" + } + } + } + } + } + } + }, + "kibana": { + "properties": { + "log": { + "properties": { + "meta": { + "type": "object" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "type": "object" + }, + "container": { + "properties": { + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "logstash": { + "properties": { + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_event": { + "type": "object" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "norms": false, + "type": "text" + } + } + }, + "slowlog": { + "properties": { + "event": { + "norms": false, + "type": "text" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_params": { + "norms": false, + "type": "text" + }, + "plugin_params_object": { + "type": "object" + }, + "plugin_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "norms": false, + "type": "text" + }, + "took_in_millis": { + "type": "long" + }, + "took_in_nanos": { + "type": "long" + } + } + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mongodb": { + "properties": { + "log": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "context": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "bytes_sent": { + "type": "long" + }, + "current_user": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesort": { + "type": "boolean" + }, + "filesort_on_disk": { + "type": "boolean" + }, + "full_join": { + "type": "boolean" + }, + "full_scan": { + "type": "boolean" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "innodb": { + "properties": { + "io_r_bytes": { + "type": "long" + }, + "io_r_ops": { + "type": "long" + }, + "io_r_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "pages_distinct": { + "type": "long" + }, + "queue_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "rec_lock_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "trx_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "killed": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_errno": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "log_slow_rate_limit": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_slow_rate_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "merge_passes": { + "type": "long" + }, + "priority_queue": { + "type": "boolean" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_cache_hit": { + "type": "boolean" + }, + "query_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "rows_affected": { + "type": "long" + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "schema": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "tmp_disk_tables": { + "type": "long" + }, + "tmp_table": { + "type": "boolean" + }, + "tmp_table_on_disk": { + "type": "boolean" + }, + "tmp_table_sizes": { + "type": "long" + }, + "tmp_tables": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "netflow": { + "properties": { + "absolute_error": { + "type": "double" + }, + "address_pool_high_threshold": { + "type": "long" + }, + "address_pool_low_threshold": { + "type": "long" + }, + "address_port_mapping_high_threshold": { + "type": "long" + }, + "address_port_mapping_low_threshold": { + "type": "long" + }, + "address_port_mapping_per_user_high_threshold": { + "type": "long" + }, + "anonymization_flags": { + "type": "long" + }, + "anonymization_technique": { + "type": "long" + }, + "application_category_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_group_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_id": { + "type": "short" + }, + "application_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_sub_category_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "bgp_destination_as_number": { + "type": "long" + }, + "bgp_next_adjacent_as_number": { + "type": "long" + }, + "bgp_next_hop_ipv4_address": { + "type": "ip" + }, + "bgp_next_hop_ipv6_address": { + "type": "ip" + }, + "bgp_prev_adjacent_as_number": { + "type": "long" + }, + "bgp_source_as_number": { + "type": "long" + }, + "bgp_validity_state": { + "type": "short" + }, + "biflow_direction": { + "type": "short" + }, + "class_id": { + "type": "short" + }, + "class_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification_engine_id": { + "type": "short" + }, + "collection_time_milliseconds": { + "type": "date" + }, + "collector_certificate": { + "type": "short" + }, + "collector_ipv4_address": { + "type": "ip" + }, + "collector_ipv6_address": { + "type": "ip" + }, + "collector_transport_port": { + "type": "long" + }, + "common_properties_id": { + "type": "long" + }, + "confidence_level": { + "type": "double" + }, + "connection_sum_duration_seconds": { + "type": "long" + }, + "connection_transaction_id": { + "type": "long" + }, + "data_link_frame_section": { + "type": "short" + }, + "data_link_frame_size": { + "type": "long" + }, + "data_link_frame_type": { + "type": "long" + }, + "data_records_reliability": { + "type": "boolean" + }, + "delta_flow_count": { + "type": "long" + }, + "destination_ipv4_address": { + "type": "ip" + }, + "destination_ipv4_prefix": { + "type": "ip" + }, + "destination_ipv4_prefix_length": { + "type": "short" + }, + "destination_ipv6_address": { + "type": "ip" + }, + "destination_ipv6_prefix": { + "type": "ip" + }, + "destination_ipv6_prefix_length": { + "type": "short" + }, + "destination_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination_transport_port": { + "type": "long" + }, + "digest_hash_value": { + "type": "long" + }, + "distinct_count_of_destinatio_nipa_ddress": { + "type": "long" + }, + "distinct_count_of_destination_ipv4_address": { + "type": "long" + }, + "distinct_count_of_destination_ipv6_address": { + "type": "long" + }, + "distinct_count_of_sourc_eipa_ddress": { + "type": "long" + }, + "distinct_count_of_source_ipv4_address": { + "type": "long" + }, + "distinct_count_of_source_ipv6_address": { + "type": "long" + }, + "dot1q_customer_dei": { + "type": "boolean" + }, + "dot1q_customer_destination_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "dot1q_customer_priority": { + "type": "short" + }, + "dot1q_customer_source_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "dot1q_customer_vlan_id": { + "type": "long" + }, + "dot1q_dei": { + "type": "boolean" + }, + "dot1q_priority": { + "type": "short" + }, + "dot1q_service_instance_id": { + "type": "long" + }, + "dot1q_service_instance_priority": { + "type": "short" + }, + "dot1q_service_instance_tag": { + "type": "short" + }, + "dot1q_vlan_id": { + "type": "long" + }, + "dropped_layer2_octet_delta_count": { + "type": "long" + }, + "dropped_layer2_octet_total_count": { + "type": "long" + }, + "dropped_octet_delta_count": { + "type": "long" + }, + "dropped_octet_total_count": { + "type": "long" + }, + "dropped_packet_delta_count": { + "type": "long" + }, + "dropped_packet_total_count": { + "type": "long" + }, + "dst_traffic_index": { + "type": "long" + }, + "egress_broadcast_packet_total_count": { + "type": "long" + }, + "egress_interface": { + "type": "long" + }, + "egress_interface_type": { + "type": "long" + }, + "egress_physical_interface": { + "type": "long" + }, + "egress_unicast_packet_total_count": { + "type": "long" + }, + "egress_vrfid": { + "type": "long" + }, + "encrypted_technology": { + "ignore_above": 1024, + "type": "keyword" + }, + "engine_id": { + "type": "short" + }, + "engine_type": { + "type": "short" + }, + "ethernet_header_length": { + "type": "short" + }, + "ethernet_payload_length": { + "type": "long" + }, + "ethernet_total_length": { + "type": "long" + }, + "ethernet_type": { + "type": "long" + }, + "export_interface": { + "type": "long" + }, + "export_protocol_version": { + "type": "short" + }, + "export_sctp_stream_id": { + "type": "long" + }, + "export_transport_protocol": { + "type": "short" + }, + "exported_flow_record_total_count": { + "type": "long" + }, + "exported_message_total_count": { + "type": "long" + }, + "exported_octet_total_count": { + "type": "long" + }, + "exporter": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_id": { + "type": "long" + }, + "timestamp": { + "type": "date" + }, + "uptime_millis": { + "type": "long" + }, + "version": { + "type": "long" + } + } + }, + "exporter_certificate": { + "type": "short" + }, + "exporter_ipv4_address": { + "type": "ip" + }, + "exporter_ipv6_address": { + "type": "ip" + }, + "exporter_transport_port": { + "type": "long" + }, + "exporting_process_id": { + "type": "long" + }, + "external_address_realm": { + "type": "short" + }, + "firewall_event": { + "type": "short" + }, + "flags_and_sampler_id": { + "type": "long" + }, + "flow_active_timeout": { + "type": "long" + }, + "flow_direction": { + "type": "short" + }, + "flow_duration_microseconds": { + "type": "long" + }, + "flow_duration_milliseconds": { + "type": "long" + }, + "flow_end_delta_microseconds": { + "type": "long" + }, + "flow_end_microseconds": { + "type": "date" + }, + "flow_end_milliseconds": { + "type": "date" + }, + "flow_end_nanoseconds": { + "type": "date" + }, + "flow_end_reason": { + "type": "short" + }, + "flow_end_seconds": { + "type": "date" + }, + "flow_end_sys_up_time": { + "type": "long" + }, + "flow_id": { + "type": "long" + }, + "flow_idle_timeout": { + "type": "long" + }, + "flow_key_indicator": { + "type": "long" + }, + "flow_label_ipv6": { + "type": "long" + }, + "flow_sampling_time_interval": { + "type": "long" + }, + "flow_sampling_time_spacing": { + "type": "long" + }, + "flow_selected_flow_delta_count": { + "type": "long" + }, + "flow_selected_octet_delta_count": { + "type": "long" + }, + "flow_selected_packet_delta_count": { + "type": "long" + }, + "flow_selector_algorithm": { + "type": "long" + }, + "flow_start_delta_microseconds": { + "type": "long" + }, + "flow_start_microseconds": { + "type": "date" + }, + "flow_start_milliseconds": { + "type": "date" + }, + "flow_start_nanoseconds": { + "type": "date" + }, + "flow_start_seconds": { + "type": "date" + }, + "flow_start_sys_up_time": { + "type": "long" + }, + "forwarding_status": { + "type": "short" + }, + "fragment_flags": { + "type": "short" + }, + "fragment_identification": { + "type": "long" + }, + "fragment_offset": { + "type": "long" + }, + "global_address_mapping_high_threshold": { + "type": "long" + }, + "gre_key": { + "type": "long" + }, + "hash_digest_output": { + "type": "boolean" + }, + "hash_flow_domain": { + "type": "long" + }, + "hash_initialiser_value": { + "type": "long" + }, + "hash_ipp_ayload_offset": { + "type": "long" + }, + "hash_ipp_ayload_size": { + "type": "long" + }, + "hash_output_range_max": { + "type": "long" + }, + "hash_output_range_min": { + "type": "long" + }, + "hash_selected_range_max": { + "type": "long" + }, + "hash_selected_range_min": { + "type": "long" + }, + "http_content_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_message_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_reason_phrase": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_request_host": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_request_method": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_request_target": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_status_code": { + "type": "long" + }, + "http_user_agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp_code_ipv4": { + "type": "short" + }, + "icmp_code_ipv6": { + "type": "short" + }, + "icmp_type_code_ipv4": { + "type": "long" + }, + "icmp_type_code_ipv6": { + "type": "long" + }, + "icmp_type_ipv4": { + "type": "short" + }, + "icmp_type_ipv6": { + "type": "short" + }, + "igmp_type": { + "type": "short" + }, + "ignored_data_record_total_count": { + "type": "long" + }, + "ignored_layer2_frame_total_count": { + "type": "long" + }, + "ignored_layer2_octet_total_count": { + "type": "long" + }, + "ignored_octet_total_count": { + "type": "long" + }, + "ignored_packet_total_count": { + "type": "long" + }, + "information_element_data_type": { + "type": "short" + }, + "information_element_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "information_element_id": { + "type": "long" + }, + "information_element_index": { + "type": "long" + }, + "information_element_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "information_element_range_begin": { + "type": "long" + }, + "information_element_range_end": { + "type": "long" + }, + "information_element_semantics": { + "type": "short" + }, + "information_element_units": { + "type": "long" + }, + "ingress_broadcast_packet_total_count": { + "type": "long" + }, + "ingress_interface": { + "type": "long" + }, + "ingress_interface_type": { + "type": "long" + }, + "ingress_multicast_packet_total_count": { + "type": "long" + }, + "ingress_physical_interface": { + "type": "long" + }, + "ingress_unicast_packet_total_count": { + "type": "long" + }, + "ingress_vrfid": { + "type": "long" + }, + "initiator_octets": { + "type": "long" + }, + "initiator_packets": { + "type": "long" + }, + "interface_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "interface_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "intermediate_process_id": { + "type": "long" + }, + "internal_address_realm": { + "type": "short" + }, + "ip_class_of_service": { + "type": "short" + }, + "ip_diff_serv_code_point": { + "type": "short" + }, + "ip_header_length": { + "type": "short" + }, + "ip_header_packet_section": { + "type": "short" + }, + "ip_next_hop_ipv4_address": { + "type": "ip" + }, + "ip_next_hop_ipv6_address": { + "type": "ip" + }, + "ip_payload_length": { + "type": "long" + }, + "ip_payload_packet_section": { + "type": "short" + }, + "ip_precedence": { + "type": "short" + }, + "ip_sec_spi": { + "type": "long" + }, + "ip_total_length": { + "type": "long" + }, + "ip_ttl": { + "type": "short" + }, + "ip_version": { + "type": "short" + }, + "ipv4_ihl": { + "type": "short" + }, + "ipv4_options": { + "type": "long" + }, + "ipv4_router_sc": { + "type": "ip" + }, + "ipv6_extension_headers": { + "type": "long" + }, + "is_multicast": { + "type": "short" + }, + "layer2_frame_delta_count": { + "type": "long" + }, + "layer2_frame_total_count": { + "type": "long" + }, + "layer2_octet_delta_count": { + "type": "long" + }, + "layer2_octet_delta_sum_of_squares": { + "type": "long" + }, + "layer2_octet_total_count": { + "type": "long" + }, + "layer2_octet_total_sum_of_squares": { + "type": "long" + }, + "layer2_segment_id": { + "type": "long" + }, + "layer2packet_section_data": { + "type": "short" + }, + "layer2packet_section_offset": { + "type": "long" + }, + "layer2packet_section_size": { + "type": "long" + }, + "line_card_id": { + "type": "long" + }, + "lower_cli_imit": { + "type": "double" + }, + "max_bieb_ntries": { + "type": "long" + }, + "max_entries_per_user": { + "type": "long" + }, + "max_export_seconds": { + "type": "date" + }, + "max_flow_end_microseconds": { + "type": "date" + }, + "max_flow_end_milliseconds": { + "type": "date" + }, + "max_flow_end_nanoseconds": { + "type": "date" + }, + "max_flow_end_seconds": { + "type": "date" + }, + "max_fragments_pending_reassembly": { + "type": "long" + }, + "max_session_entries": { + "type": "long" + }, + "max_subscribers": { + "type": "long" + }, + "maximum_ip_total_length": { + "type": "long" + }, + "maximum_layer2_total_length": { + "type": "long" + }, + "maximum_ttl": { + "type": "short" + }, + "message_md5_checksum": { + "type": "short" + }, + "message_scope": { + "type": "short" + }, + "metering_process_id": { + "type": "long" + }, + "metro_evc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "metro_evc_type": { + "type": "short" + }, + "mib_capture_time_semantics": { + "type": "short" + }, + "mib_context_engine_id": { + "type": "short" + }, + "mib_context_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_index_indicator": { + "type": "long" + }, + "mib_module_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_identifier": { + "type": "short" + }, + "mib_object_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_syntax": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_value_bits": { + "type": "short" + }, + "mib_object_value_counter": { + "type": "long" + }, + "mib_object_value_gauge": { + "type": "long" + }, + "mib_object_value_integer": { + "type": "long" + }, + "mib_object_value_octet_string": { + "type": "short" + }, + "mib_object_value_oid": { + "type": "short" + }, + "mib_object_value_time_ticks": { + "type": "long" + }, + "mib_object_value_unsigned": { + "type": "long" + }, + "mib_object_valuei_pa_ddress": { + "type": "ip" + }, + "mib_sub_identifier": { + "type": "long" + }, + "min_export_seconds": { + "type": "date" + }, + "min_flow_start_microseconds": { + "type": "date" + }, + "min_flow_start_milliseconds": { + "type": "date" + }, + "min_flow_start_nanoseconds": { + "type": "date" + }, + "min_flow_start_seconds": { + "type": "date" + }, + "minimum_ip_total_length": { + "type": "long" + }, + "minimum_layer2_total_length": { + "type": "long" + }, + "minimum_ttl": { + "type": "short" + }, + "mobile_imsi": { + "ignore_above": 1024, + "type": "keyword" + }, + "mobile_msisdn": { + "ignore_above": 1024, + "type": "keyword" + }, + "monitoring_interval_end_milli_seconds": { + "type": "date" + }, + "monitoring_interval_start_milli_seconds": { + "type": "date" + }, + "mpls_label_stack_depth": { + "type": "long" + }, + "mpls_label_stack_length": { + "type": "long" + }, + "mpls_label_stack_section": { + "type": "short" + }, + "mpls_label_stack_section10": { + "type": "short" + }, + "mpls_label_stack_section2": { + "type": "short" + }, + "mpls_label_stack_section3": { + "type": "short" + }, + "mpls_label_stack_section4": { + "type": "short" + }, + "mpls_label_stack_section5": { + "type": "short" + }, + "mpls_label_stack_section6": { + "type": "short" + }, + "mpls_label_stack_section7": { + "type": "short" + }, + "mpls_label_stack_section8": { + "type": "short" + }, + "mpls_label_stack_section9": { + "type": "short" + }, + "mpls_payload_length": { + "type": "long" + }, + "mpls_payload_packet_section": { + "type": "short" + }, + "mpls_top_label_exp": { + "type": "short" + }, + "mpls_top_label_ipv4_address": { + "type": "ip" + }, + "mpls_top_label_ipv6_address": { + "type": "ip" + }, + "mpls_top_label_prefix_length": { + "type": "short" + }, + "mpls_top_label_stack_section": { + "type": "short" + }, + "mpls_top_label_ttl": { + "type": "short" + }, + "mpls_top_label_type": { + "type": "short" + }, + "mpls_vpn_route_distinguisher": { + "type": "short" + }, + "multicast_replication_factor": { + "type": "long" + }, + "nat_event": { + "type": "short" + }, + "nat_instance_id": { + "type": "long" + }, + "nat_originating_address_realm": { + "type": "short" + }, + "nat_pool_id": { + "type": "long" + }, + "nat_pool_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat_quota_exceeded_event": { + "type": "long" + }, + "nat_threshold_event": { + "type": "long" + }, + "nat_type": { + "type": "short" + }, + "new_connection_delta_count": { + "type": "long" + }, + "next_header_ipv6": { + "type": "short" + }, + "not_sent_flow_total_count": { + "type": "long" + }, + "not_sent_layer2_octet_total_count": { + "type": "long" + }, + "not_sent_octet_total_count": { + "type": "long" + }, + "not_sent_packet_total_count": { + "type": "long" + }, + "observation_domain_id": { + "type": "long" + }, + "observation_domain_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "observation_point_id": { + "type": "long" + }, + "observation_point_type": { + "type": "short" + }, + "observation_time_microseconds": { + "type": "date" + }, + "observation_time_milliseconds": { + "type": "date" + }, + "observation_time_nanoseconds": { + "type": "date" + }, + "observation_time_seconds": { + "type": "date" + }, + "observed_flow_total_count": { + "type": "long" + }, + "octet_delta_count": { + "type": "long" + }, + "octet_delta_sum_of_squares": { + "type": "long" + }, + "octet_total_count": { + "type": "long" + }, + "octet_total_sum_of_squares": { + "type": "long" + }, + "opaque_octets": { + "type": "short" + }, + "original_exporter_ipv4_address": { + "type": "ip" + }, + "original_exporter_ipv6_address": { + "type": "ip" + }, + "original_flows_completed": { + "type": "long" + }, + "original_flows_initiated": { + "type": "long" + }, + "original_flows_present": { + "type": "long" + }, + "original_observation_domain_id": { + "type": "long" + }, + "p2p_technology": { + "ignore_above": 1024, + "type": "keyword" + }, + "packet_delta_count": { + "type": "long" + }, + "packet_total_count": { + "type": "long" + }, + "padding_octets": { + "type": "short" + }, + "payload_length_ipv6": { + "type": "long" + }, + "port_id": { + "type": "long" + }, + "port_range_end": { + "type": "long" + }, + "port_range_num_ports": { + "type": "long" + }, + "port_range_start": { + "type": "long" + }, + "port_range_step_size": { + "type": "long" + }, + "post_destination_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "post_dot1q_customer_vlan_id": { + "type": "long" + }, + "post_dot1q_vlan_id": { + "type": "long" + }, + "post_ip_class_of_service": { + "type": "short" + }, + "post_ip_diff_serv_code_point": { + "type": "short" + }, + "post_ip_precedence": { + "type": "short" + }, + "post_layer2_octet_delta_count": { + "type": "long" + }, + "post_layer2_octet_total_count": { + "type": "long" + }, + "post_mcast_layer2_octet_delta_count": { + "type": "long" + }, + "post_mcast_layer2_octet_total_count": { + "type": "long" + }, + "post_mcast_octet_delta_count": { + "type": "long" + }, + "post_mcast_octet_total_count": { + "type": "long" + }, + "post_mcast_packet_delta_count": { + "type": "long" + }, + "post_mcast_packet_total_count": { + "type": "long" + }, + "post_mpls_top_label_exp": { + "type": "short" + }, + "post_nadt_estination_ipv4_address": { + "type": "ip" + }, + "post_nadt_estination_ipv6_address": { + "type": "ip" + }, + "post_napdt_estination_transport_port": { + "type": "long" + }, + "post_napst_ource_transport_port": { + "type": "long" + }, + "post_nast_ource_ipv4_address": { + "type": "ip" + }, + "post_nast_ource_ipv6_address": { + "type": "ip" + }, + "post_octet_delta_count": { + "type": "long" + }, + "post_octet_total_count": { + "type": "long" + }, + "post_packet_delta_count": { + "type": "long" + }, + "post_packet_total_count": { + "type": "long" + }, + "post_source_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "post_vlan_id": { + "type": "long" + }, + "private_enterprise_number": { + "type": "long" + }, + "protocol_identifier": { + "type": "short" + }, + "pseudo_wire_control_word": { + "type": "long" + }, + "pseudo_wire_destination_ipv4_address": { + "type": "ip" + }, + "pseudo_wire_id": { + "type": "long" + }, + "pseudo_wire_type": { + "type": "long" + }, + "relative_error": { + "type": "double" + }, + "responder_octets": { + "type": "long" + }, + "responder_packets": { + "type": "long" + }, + "rfc3550_jitter_microseconds": { + "type": "long" + }, + "rfc3550_jitter_milliseconds": { + "type": "long" + }, + "rfc3550_jitter_nanoseconds": { + "type": "long" + }, + "rtp_sequence_number": { + "type": "long" + }, + "sampler_id": { + "type": "short" + }, + "sampler_mode": { + "type": "short" + }, + "sampler_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sampler_random_interval": { + "type": "long" + }, + "sampling_algorithm": { + "type": "short" + }, + "sampling_flow_interval": { + "type": "long" + }, + "sampling_flow_spacing": { + "type": "long" + }, + "sampling_interval": { + "type": "long" + }, + "sampling_packet_interval": { + "type": "long" + }, + "sampling_packet_space": { + "type": "long" + }, + "sampling_population": { + "type": "long" + }, + "sampling_probability": { + "type": "double" + }, + "sampling_size": { + "type": "long" + }, + "sampling_time_interval": { + "type": "long" + }, + "sampling_time_space": { + "type": "long" + }, + "section_exported_octets": { + "type": "long" + }, + "section_offset": { + "type": "long" + }, + "selection_sequence_id": { + "type": "long" + }, + "selector_algorithm": { + "type": "long" + }, + "selector_id": { + "type": "long" + }, + "selector_id_total_pkts_observed": { + "type": "long" + }, + "selector_id_total_pkts_selected": { + "type": "long" + }, + "selector_itd_otal_flows_observed": { + "type": "long" + }, + "selector_itd_otal_flows_selected": { + "type": "long" + }, + "selector_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "session_scope": { + "type": "short" + }, + "source_ipv4_address": { + "type": "ip" + }, + "source_ipv4_prefix": { + "type": "ip" + }, + "source_ipv4_prefix_length": { + "type": "short" + }, + "source_ipv6_address": { + "type": "ip" + }, + "source_ipv6_prefix": { + "type": "ip" + }, + "source_ipv6_prefix_length": { + "type": "short" + }, + "source_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_transport_port": { + "type": "long" + }, + "source_transport_ports_limit": { + "type": "long" + }, + "src_traffic_index": { + "type": "long" + }, + "sta_ipv4_address": { + "type": "ip" + }, + "sta_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "system_init_time_milliseconds": { + "type": "date" + }, + "tcp_ack_total_count": { + "type": "long" + }, + "tcp_acknowledgement_number": { + "type": "long" + }, + "tcp_control_bits": { + "type": "long" + }, + "tcp_destination_port": { + "type": "long" + }, + "tcp_fin_total_count": { + "type": "long" + }, + "tcp_header_length": { + "type": "short" + }, + "tcp_options": { + "type": "long" + }, + "tcp_psh_total_count": { + "type": "long" + }, + "tcp_rst_total_count": { + "type": "long" + }, + "tcp_sequence_number": { + "type": "long" + }, + "tcp_source_port": { + "type": "long" + }, + "tcp_syn_total_count": { + "type": "long" + }, + "tcp_urg_total_count": { + "type": "long" + }, + "tcp_urgent_pointer": { + "type": "long" + }, + "tcp_window_scale": { + "type": "long" + }, + "tcp_window_size": { + "type": "long" + }, + "template_id": { + "type": "long" + }, + "total_length_ipv4": { + "type": "long" + }, + "transport_octet_delta_count": { + "type": "long" + }, + "transport_packet_delta_count": { + "type": "long" + }, + "tunnel_technology": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "udp_destination_port": { + "type": "long" + }, + "udp_message_length": { + "type": "long" + }, + "udp_source_port": { + "type": "long" + }, + "upper_cli_imit": { + "type": "double" + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "value_distribution_method": { + "type": "short" + }, + "virtual_station_interface_id": { + "type": "short" + }, + "virtual_station_interface_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "virtual_station_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "virtual_station_uuid": { + "type": "short" + }, + "vlan_id": { + "type": "long" + }, + "vpn_identifier": { + "type": "short" + }, + "vr_fname": { + "ignore_above": 1024, + "type": "keyword" + }, + "wlan_channel_id": { + "type": "short" + }, + "wlan_ssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "wtp_mac_address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "network": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "index": false, + "norms": false, + "type": "text" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, + "offset": { + "type": "long" + }, + "osquery": { + "properties": { + "result": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "calendar_time": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "unix_time": { + "type": "long" + } + } + } + } + }, + "postgresql": { + "properties": { + "log": { + "properties": { + "core_id": { + "type": "long" + }, + "database": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "float" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "process": { + "properties": { + "pid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "prospector": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "read_timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "redis": { + "properties": { + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_ecs": { + "properties": { + "bytes": { + "type": "long" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "stream": { + "ignore_above": 1024, + "type": "keyword" + }, + "suricata": { + "properties": { + "eve": { + "properties": { + "alert": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "type": "long" + }, + "rev": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_id": { + "type": "long" + } + } + }, + "app_proto": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_proto_expected": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_proto_orig": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_proto_tc": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_proto_ts": { + "ignore_above": 1024, + "type": "keyword" + }, + "dest_ip": { + "type": "ip" + }, + "dest_port": { + "type": "long" + }, + "dns": { + "properties": { + "id": { + "type": "long" + }, + "rcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "rdata": { + "ignore_above": 1024, + "type": "keyword" + }, + "rrname": { + "ignore_above": 1024, + "type": "keyword" + }, + "rrtype": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "tx_id": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "properties": { + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileinfo": { + "properties": { + "filename": { + "ignore_above": 1024, + "type": "keyword" + }, + "gaps": { + "type": "boolean" + }, + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "stored": { + "type": "boolean" + }, + "tx_id": { + "type": "long" + } + } + }, + "flags": { + "properties": {} + }, + "flow": { + "properties": { + "age": { + "type": "long" + }, + "alerted": { + "type": "boolean" + }, + "bytes_toclient": { + "type": "long" + }, + "bytes_toserver": { + "type": "long" + }, + "end": { + "type": "date" + }, + "pkts_toclient": { + "type": "long" + }, + "pkts_toserver": { + "type": "long" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flow_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "http": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_content_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_method": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_refer": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_user_agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "redirect": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "icmp_code": { + "type": "long" + }, + "icmp_type": { + "type": "long" + }, + "in_iface": { + "ignore_above": 1024, + "type": "keyword" + }, + "pcap_cnt": { + "type": "long" + }, + "proto": { + "ignore_above": 1024, + "type": "keyword" + }, + "smtp": { + "properties": { + "helo": { + "ignore_above": 1024, + "type": "keyword" + }, + "mail_from": { + "ignore_above": 1024, + "type": "keyword" + }, + "rcpt_to": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "src_ip": { + "type": "ip" + }, + "src_port": { + "type": "long" + }, + "ssh": { + "properties": { + "client": { + "properties": { + "proto_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "software_version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "proto_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "software_version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "stats": { + "properties": { + "app_layer": { + "properties": { + "flow": { + "properties": { + "dcerpc_tcp": { + "type": "long" + }, + "dcerpc_udp": { + "type": "long" + }, + "dns_tcp": { + "type": "long" + }, + "dns_udp": { + "type": "long" + }, + "failed_tcp": { + "type": "long" + }, + "failed_udp": { + "type": "long" + }, + "ftp": { + "type": "long" + }, + "http": { + "type": "long" + }, + "imap": { + "type": "long" + }, + "msn": { + "type": "long" + }, + "smb": { + "type": "long" + }, + "smtp": { + "type": "long" + }, + "ssh": { + "type": "long" + }, + "tls": { + "type": "long" + } + } + }, + "tx": { + "properties": { + "dcerpc_tcp": { + "type": "long" + }, + "dcerpc_udp": { + "type": "long" + }, + "dns_tcp": { + "type": "long" + }, + "dns_udp": { + "type": "long" + }, + "ftp": { + "type": "long" + }, + "http": { + "type": "long" + }, + "smb": { + "type": "long" + }, + "smtp": { + "type": "long" + }, + "ssh": { + "type": "long" + }, + "tls": { + "type": "long" + } + } + } + } + }, + "capture": { + "properties": { + "kernel_drops": { + "type": "long" + }, + "kernel_ifdrops": { + "type": "long" + }, + "kernel_packets": { + "type": "long" + } + } + }, + "decoder": { + "properties": { + "avg_pkt_size": { + "type": "long" + }, + "bytes": { + "type": "long" + }, + "dce": { + "properties": { + "pkt_too_small": { + "type": "long" + } + } + }, + "erspan": { + "type": "long" + }, + "ethernet": { + "type": "long" + }, + "gre": { + "type": "long" + }, + "icmpv4": { + "type": "long" + }, + "icmpv6": { + "type": "long" + }, + "ieee8021ah": { + "type": "long" + }, + "invalid": { + "type": "long" + }, + "ipraw": { + "properties": { + "invalid_ip_version": { + "type": "long" + } + } + }, + "ipv4": { + "type": "long" + }, + "ipv4_in_ipv6": { + "type": "long" + }, + "ipv6": { + "type": "long" + }, + "ipv6_in_ipv6": { + "type": "long" + }, + "ltnull": { + "properties": { + "pkt_too_small": { + "type": "long" + }, + "unsupported_type": { + "type": "long" + } + } + }, + "max_pkt_size": { + "type": "long" + }, + "mpls": { + "type": "long" + }, + "null": { + "type": "long" + }, + "pkts": { + "type": "long" + }, + "ppp": { + "type": "long" + }, + "pppoe": { + "type": "long" + }, + "raw": { + "type": "long" + }, + "sctp": { + "type": "long" + }, + "sll": { + "type": "long" + }, + "tcp": { + "type": "long" + }, + "teredo": { + "type": "long" + }, + "udp": { + "type": "long" + }, + "vlan": { + "type": "long" + }, + "vlan_qinq": { + "type": "long" + } + } + }, + "defrag": { + "properties": { + "ipv4": { + "properties": { + "fragments": { + "type": "long" + }, + "reassembled": { + "type": "long" + }, + "timeouts": { + "type": "long" + } + } + }, + "ipv6": { + "properties": { + "fragments": { + "type": "long" + }, + "reassembled": { + "type": "long" + }, + "timeouts": { + "type": "long" + } + } + }, + "max_frag_hits": { + "type": "long" + } + } + }, + "detect": { + "properties": { + "alert": { + "type": "long" + } + } + }, + "dns": { + "properties": { + "memcap_global": { + "type": "long" + }, + "memcap_state": { + "type": "long" + }, + "memuse": { + "type": "long" + } + } + }, + "file_store": { + "properties": { + "open_files": { + "type": "long" + } + } + }, + "flow": { + "properties": { + "emerg_mode_entered": { + "type": "long" + }, + "emerg_mode_over": { + "type": "long" + }, + "icmpv4": { + "type": "long" + }, + "icmpv6": { + "type": "long" + }, + "memcap": { + "type": "long" + }, + "memuse": { + "type": "long" + }, + "spare": { + "type": "long" + }, + "tcp": { + "type": "long" + }, + "tcp_reuse": { + "type": "long" + }, + "udp": { + "type": "long" + } + } + }, + "flow_mgr": { + "properties": { + "bypassed_pruned": { + "type": "long" + }, + "closed_pruned": { + "type": "long" + }, + "est_pruned": { + "type": "long" + }, + "flows_checked": { + "type": "long" + }, + "flows_notimeout": { + "type": "long" + }, + "flows_removed": { + "type": "long" + }, + "flows_timeout": { + "type": "long" + }, + "flows_timeout_inuse": { + "type": "long" + }, + "new_pruned": { + "type": "long" + }, + "rows_busy": { + "type": "long" + }, + "rows_checked": { + "type": "long" + }, + "rows_empty": { + "type": "long" + }, + "rows_maxlen": { + "type": "long" + }, + "rows_skipped": { + "type": "long" + } + } + }, + "http": { + "properties": { + "memcap": { + "type": "long" + }, + "memuse": { + "type": "long" + } + } + }, + "tcp": { + "properties": { + "insert_data_normal_fail": { + "type": "long" + }, + "insert_data_overlap_fail": { + "type": "long" + }, + "insert_list_fail": { + "type": "long" + }, + "invalid_checksum": { + "type": "long" + }, + "memuse": { + "type": "long" + }, + "no_flow": { + "type": "long" + }, + "overlap": { + "type": "long" + }, + "overlap_diff_data": { + "type": "long" + }, + "pseudo": { + "type": "long" + }, + "pseudo_failed": { + "type": "long" + }, + "reassembly_gap": { + "type": "long" + }, + "reassembly_memuse": { + "type": "long" + }, + "rst": { + "type": "long" + }, + "segment_memcap_drop": { + "type": "long" + }, + "sessions": { + "type": "long" + }, + "ssn_memcap_drop": { + "type": "long" + }, + "stream_depth_reached": { + "type": "long" + }, + "syn": { + "type": "long" + }, + "synack": { + "type": "long" + } + } + }, + "uptime": { + "type": "long" + } + } + }, + "tcp": { + "properties": { + "ack": { + "type": "boolean" + }, + "fin": { + "type": "boolean" + }, + "psh": { + "type": "boolean" + }, + "rst": { + "type": "boolean" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "syn": { + "type": "boolean" + }, + "tcp_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_flags_tc": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_flags_ts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "type": "date" + }, + "tls": { + "properties": { + "fingerprint": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuerdn": { + "ignore_above": 1024, + "type": "keyword" + }, + "notafter": { + "type": "date" + }, + "notbefore": { + "type": "date" + }, + "serial": { + "ignore_above": 1024, + "type": "keyword" + }, + "session_resumed": { + "type": "boolean" + }, + "sni": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tx_id": { + "type": "long" + } + } + } + } + }, + "syslog": { + "properties": { + "facility": { + "type": "long" + }, + "facility_label": { + "ignore_above": 1024, + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "severity_label": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "properties": { + "gid": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "event": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "useradd": { + "properties": { + "gid": { + "type": "long" + }, + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "type": "long" + } + } + } + } + }, + "syslog": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "traefik": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "backend_url": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "duration": { + "type": "long" + }, + "frontend_name": { + "norms": false, + "type": "text" + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_count": { + "type": "long" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "index": false, + "norms": false, + "type": "text" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } +} diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index 3bddc60b36449..cb29695e306a8 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -25,11 +25,13 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CloseIndexRequest; +import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.searchable_snapshots.MountSnapshotRequest; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -60,7 +62,9 @@ import java.util.stream.Collectors; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; public class OldRepositoryAccessIT extends ESRestTestCase { @Override @@ -291,6 +295,40 @@ private void restoreMountAndVerify( .getStatus() ); + MappingMetadata mapping = client.indices().getMapping(new GetMappingsRequest().indices("restored_test"), RequestOptions.DEFAULT) + .mappings().get("restored_test"); + logger.info("mapping for {}: {}", mapping.type(), mapping.source().string()); + Map root = mapping.sourceAsMap(); + assertThat(root, hasKey("_meta")); + assertThat(root.get("_meta"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map meta = (Map) root.get("_meta"); + assertThat(meta, hasKey("legacy-mapping")); + assertThat(meta.get("legacy-mapping"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map legacyMapping = (Map) meta.get("legacy-mapping"); + assertThat(legacyMapping, hasKey("properties")); + assertThat(legacyMapping.get("properties"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map propertiesMapping = (Map) legacyMapping.get("properties"); + assertThat(propertiesMapping, hasKey("val")); + assertThat(propertiesMapping.get("val"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map valMapping = (Map) propertiesMapping.get("val"); + assertThat(valMapping, hasKey("type")); + assertEquals("long", valMapping.get("type")); + +// assertThat(root, hasKey("runtime")); +// assertThat(root.get("runtime"), instanceOf(Map.class)); +// @SuppressWarnings("unchecked") +// Map runtimeMapping = (Map) root.get("runtime"); +// assertThat(runtimeMapping, hasKey("val")); +// assertThat(runtimeMapping.get("val"), instanceOf(Map.class)); +// @SuppressWarnings("unchecked") +// Map valRuntimeMapping = (Map) runtimeMapping.get("val"); +// assertThat(valRuntimeMapping, hasKey("type")); +// assertEquals("long", valRuntimeMapping.get("type")); + // run a search against the index assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository); From 05e5183d6c7af7a38af90d107d5454799dea900a Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 25 Jan 2022 10:03:45 +0100 Subject: [PATCH 2/8] do mapping-related transformations later --- .../cluster/metadata/IndexMetadata.java | 22 - .../index/mapper/LegacyMappingConverter.java | 223 - .../snapshots/RestoreService.java | 2 +- .../mapper/LegacyMappingConverterTests.java | 48 - .../elasticsearch/index/mapper/filebeat5.json | 683 --- .../elasticsearch/index/mapper/filebeat6.json | 4903 ----------------- .../oldrepos/OldRepositoryAccessIT.java | 13 +- 7 files changed, 2 insertions(+), 5892 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java delete mode 100644 server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json delete mode 100644 server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 32850585b3a59..71c3ec2b2f967 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -39,7 +39,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; -import org.elasticsearch.index.mapper.LegacyMappingConverter; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexLongFieldRange; @@ -61,7 +60,6 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -1981,28 +1979,8 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE } private static void handleLegacyMapping(Builder builder, Map mapping) { - //Map adaptedMapping = new - - // we don't need to obey any routing here stuff is read-only anyway and get is disabled - // final String mapping = "{ \"_doc\" : { \"enabled\": false, \"_meta\": " + mmd.source().string() + " } }"; - -// LegacyMappingConverter legacyMappingConverter = new LegacyMappingConverter(); -// @SuppressWarnings("unchecked") -// Map newMapping = (Map) legacyMappingConverter.extractNewMapping(mapping, true); -// mapping = newMapping; - if (mapping.size() == 1) { String mappingType = mapping.keySet().iterator().next(); -// @SuppressWarnings("unchecked") -// Map typelessMapping = (Map) mapping.get(mappingType); -// Map runtimeFieldMapping = legacyMappingConverter.extractRuntimeFieldMapping(typelessMapping); -// if (runtimeFieldMapping.isEmpty() == false) { -// // TODO: only add those runtime fields that don't already exist -// LinkedHashMap newTypelessMapping = new LinkedHashMap<>(); -// newTypelessMapping.put("runtime", runtimeFieldMapping); -// newTypelessMapping.putAll(typelessMapping); -// mapping.put(mappingType, newTypelessMapping); -// } builder.putMapping(new MappingMetadata(mappingType, mapping)); } else if (mapping.size() > 1) { // TODO: handle this better diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java deleted file mode 100644 index ee4a138a86e6b..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyMappingConverter.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.action.admin.cluster.stats.MappingVisitor; -import org.elasticsearch.common.xcontent.support.XContentMapValues; - -import java.util.LinkedHashMap; -import java.util.Map; - -public class LegacyMappingConverter { - - public Map extractRuntimeFieldMapping(Map mapping) { - final Map result = new LinkedHashMap<>(); - MappingVisitor.visitMapping(mapping, (key, map) -> { - Object type = map.get("type"); - if (type instanceof String) { - String typeAsString = (String) type; - // copy over known basic types for which there are runtime fields - // first auto-convert some types - switch (typeAsString) { - case "byte": - case "short": - case "integer": - typeAsString = "long"; - break; - case "half_float": - case "float": - typeAsString = "double"; - break; - default: - // ignore - break; - } - switch (typeAsString) { - case "boolean": - case "date": - case "double": - case "geo_point": - case "ip": - case "keyword": - case "long": - // copy over only format parameter if it exists - // TODO: converting multi-fields to default runtime fields does not make sense - // (as field name in mapping is not equal to field name in _source) - // This means we need to add a script that returns the correct field from _source - final Map runtimeFieldDefinition = new LinkedHashMap<>(); - runtimeFieldDefinition.put("type", typeAsString); - Object format = map.get("format"); - if (format instanceof String) { - runtimeFieldDefinition.put("format", format); - } - result.put(key, runtimeFieldDefinition); - break; - default: - // ignore - break; - } - } - }); - return result; - } - - public Map extractDocValuesMapping(Map mapping) { - final Map result = new LinkedHashMap<>(); - MappingVisitor.visitMapping(mapping, (key, map) -> { - Object type = map.get("type"); - if (type instanceof String) { - // copy over known basic types for which there are doc-value only implementations - switch ((String) type) { - case "date": - case "date_nanos": - case "long": - case "integer": - case "short": - case "byte": - case "double": - case "float": - case "half_float": - case "scaled_float": - final Object docValues = map.get("doc_values"); - // check that doc_values had not been disabled - if (docValues != null && XContentMapValues.nodeBooleanValue(docValues) == false) { - return; - } - final Map docValueOnlyDefinition = new LinkedHashMap<>(map); - docValueOnlyDefinition.put("index", false); - // TODO: this does not work, as the key is "dotted" - // we would need to recreate hierarchical mapping - result.put(key, docValueOnlyDefinition); - break; - default: - // ignore - } - } - }); - return result; - } - - public Map extractNewMapping(Map mapping, boolean withTypes) { - final Map result = new LinkedHashMap<>(); - if (withTypes) { - for (Map.Entry entry : mapping.entrySet()) { - @SuppressWarnings("unchecked") - Map actualMapping = (Map) entry.getValue(); - result.put(entry.getKey(), extractNewMapping(actualMapping, false)); - } - return result; - } - - Object properties = mapping.get("properties"); - if (properties instanceof Map) { - @SuppressWarnings("unchecked") - Map propertiesAsMap = (Map) properties; - Map newPropertiesMap = new LinkedHashMap<>(); - for (Map.Entry entry : propertiesAsMap.entrySet()) { - final Object v = entry.getValue(); - if (v instanceof Map) { - @SuppressWarnings("unchecked") - Map fieldMapping = (Map) v; - Object type = fieldMapping.get("type"); - Map fieldMap = new LinkedHashMap<>(); - if (type instanceof String) { - // copy over known types - switch ((String) type) { - case "binary": - case "boolean": - case "keyword": - case "long": - case "integer": - case "short": - case "byte": - case "double": - case "float": - case "half_float": - case "scaled_float": - case "unsigned_long": - case "date": - case "date_nanos": - case "alias": - case "object": - case "flattened": - case "nested": - case "join": - case "integer_range": - case "float_range": - case "long_range": - case "double_range": - case "date_range": - case "ip_range": - case "ip": - case "version": - case "murmur3": - case "aggregate_metric_double": - case "histogram": - case "text": - case "match_only_text": - case "annotated-text": - case "completion": - case "search_as_you_type": - case "token_count": - case "dense_vector": - case "sparse_vector": - case "rank_feature": - case "rank_features": - case "geo_point": - case "geo_shape": - case "point": - case "shape": - case "percolator": - fieldMap.putAll(fieldMapping); - break; - default: - assert false; - // ignore - } - } - - // Multi fields - Object fieldsO = fieldMapping.get("fields"); - if (fieldsO instanceof Map) { - @SuppressWarnings("unchecked") - Map fields = (Map) fieldsO; - Map fieldsMap = new LinkedHashMap<>(); - for (Map.Entry subfieldEntry : fields.entrySet()) { - Object v2 = subfieldEntry.getValue(); - if (v2 instanceof Map) { - @SuppressWarnings("unchecked") - Map fieldMapping2 = (Map) v2; - String key = entry.getKey() + "." + subfieldEntry.getKey(); - Map extracted = extractNewMapping(fieldMapping2, false); - if (extracted.isEmpty() == false) { - fieldsMap.put(key, extracted); - } - } - } - if (fieldsMap.isEmpty() == false) { - fieldMap.put("fields", fieldsMap); - } - } - - Map extracted = extractNewMapping(fieldMapping, false); - fieldMap.putAll(extracted); - - if (fieldMap.isEmpty() == false) { - newPropertiesMap.put(entry.getKey(), fieldMap); - } - } - } - if (newPropertiesMap.isEmpty() == false) { - result.put("properties", newPropertiesMap); - } - } - - return result; - } -} diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index cae3c0e71139d..ff9e70aa533df 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1298,7 +1298,7 @@ public ClusterState execute(ClusterState currentState) { ); if (snapshotIndexMetadata.getCreationVersion().before( currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion())) { - // adapt mappings so that they can be read + // adapt index metadata so that it can be understood by current version snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata); } try { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java deleted file mode 100644 index 7338c1d327aa3..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyMappingConverterTests.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.json.JsonXContent; - -import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; - -public class LegacyMappingConverterTests extends ESTestCase { - - public void testFilebeat5() throws IOException { - LegacyMappingConverter converter = new LegacyMappingConverter(); - try (XContentParser parser = createParser(JsonXContent.jsonXContent, - copyToStringFromClasspath("/org/elasticsearch/index/mapper/filebeat5.json"))) { - Map mapping = parser.mapOrdered(); - Map newMapping = converter.extractNewMapping(mapping, true); - logger.info("New mapping: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(newMapping))); - Map runtimeFieldMapping = converter.extractRuntimeFieldMapping(newMapping); - logger.info("Runtime fields: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(runtimeFieldMapping))); - assertEquals(137, runtimeFieldMapping.keySet().size()); - } - } - - public void testFilebeat6() throws IOException { - LegacyMappingConverter converter = new LegacyMappingConverter(); - try (XContentParser parser = createParser(JsonXContent.jsonXContent, - copyToStringFromClasspath("/org/elasticsearch/index/mapper/filebeat6.json"))) { - Map mapping = parser.mapOrdered(); - Map newMapping = converter.extractNewMapping(mapping, true); - logger.info("New mapping: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(newMapping))); - Map runtimeFieldMapping = converter.extractRuntimeFieldMapping(newMapping); - logger.info("Runtime fields: {}", Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(runtimeFieldMapping))); - assertEquals(1160, runtimeFieldMapping.keySet().size()); - } - } -} diff --git a/server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json b/server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json deleted file mode 100644 index c6de6a166b115..0000000000000 --- a/server/src/test/resources/org/elasticsearch/index/mapper/filebeat5.json +++ /dev/null @@ -1,683 +0,0 @@ -{ - "_default_": { - "_meta": { - "version": "5.6.17" - }, - "date_detection": false, - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "apache2": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "client": { - "ignore_above": 1024, - "type": "keyword" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - } - } - } - } - }, - "auditd": { - "properties": { - "log": { - "properties": { - "a0": { - "ignore_above": 1024, - "type": "keyword" - }, - "acct": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "item": { - "ignore_above": 1024, - "type": "keyword" - }, - "items": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ppid": { - "ignore_above": 1024, - "type": "keyword" - }, - "record_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "res": { - "ignore_above": 1024, - "type": "keyword" - }, - "sequence": { - "type": "long" - } - } - } - } - }, - "beat": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "ignore_above": 1024, - "type": "keyword" - }, - "fileset": { - "properties": { - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "input_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "meta": { - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "instance_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "project_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "mysql": { - "properties": { - "error": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "thread_id": { - "type": "long" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "slowlog": { - "properties": { - "host": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "type": "long" - }, - "ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "lock_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "query": { - "ignore_above": 1024, - "type": "keyword" - }, - "query_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "rows_examined": { - "type": "long" - }, - "rows_sent": { - "type": "long" - }, - "timestamp": { - "type": "long" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "nginx": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "connection_id": { - "type": "long" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - } - } - } - } - }, - "offset": { - "type": "long" - }, - "read_timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "source": { - "ignore_above": 1024, - "type": "keyword" - }, - "system": { - "properties": { - "auth": { - "properties": { - "groupadd": { - "properties": { - "gid": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "ssh": { - "properties": { - "dropped_ip": { - "type": "ip" - }, - "event": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - }, - "signature": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "sudo": { - "properties": { - "command": { - "ignore_above": 1024, - "type": "keyword" - }, - "error": { - "ignore_above": 1024, - "type": "keyword" - }, - "pwd": { - "ignore_above": 1024, - "type": "keyword" - }, - "tty": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "useradd": { - "properties": { - "gid": { - "type": "long" - }, - "home": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "shell": { - "ignore_above": 1024, - "type": "keyword" - }, - "uid": { - "type": "long" - } - } - } - } - }, - "syslog": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } -} diff --git a/server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json b/server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json deleted file mode 100644 index a5d14ac75a1f7..0000000000000 --- a/server/src/test/resources/org/elasticsearch/index/mapper/filebeat6.json +++ /dev/null @@ -1,4903 +0,0 @@ -{ - "doc": { - "_meta": { - "version": "6.8.0" - }, - "date_detection": false, - "dynamic_templates": [ - { - "fields": { - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "fields.*" - } - }, - { - "docker.container.labels": { - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "docker.container.labels.*" - } - }, - { - "kibana.log.meta": { - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "kibana.log.meta.*" - } - }, - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "apache2": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "ssl": { - "properties": { - "cipher": { - "ignore_above": 1024, - "type": "keyword" - }, - "protocol": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "index": false, - "norms": false, - "type": "text" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "client": { - "ignore_above": 1024, - "type": "keyword" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - } - } - } - } - }, - "auditd": { - "properties": { - "log": { - "properties": { - "a0": { - "ignore_above": 1024, - "type": "keyword" - }, - "acct": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "item": { - "ignore_above": 1024, - "type": "keyword" - }, - "items": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_auid": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_ses": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ppid": { - "ignore_above": 1024, - "type": "keyword" - }, - "record_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "res": { - "ignore_above": 1024, - "type": "keyword" - }, - "sequence": { - "type": "long" - } - } - } - } - }, - "beat": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cloud": { - "properties": { - "availability_zone": { - "path": "meta.cloud.availability_zone", - "type": "alias" - }, - "instance": { - "properties": { - "id": { - "path": "meta.cloud.instance_id", - "type": "alias" - }, - "name": { - "path": "meta.cloud.instance_name", - "type": "alias" - } - } - }, - "machine": { - "properties": { - "type": { - "path": "meta.cloud.machine_type", - "type": "alias" - } - } - }, - "project": { - "properties": { - "id": { - "path": "meta.cloud.project_id", - "type": "alias" - } - } - }, - "provider": { - "path": "meta.cloud.provider", - "type": "alias" - }, - "region": { - "path": "meta.cloud.region", - "type": "alias" - } - } - }, - "container": { - "properties": { - "id": { - "path": "docker.container.id", - "type": "alias" - }, - "image": { - "properties": { - "name": { - "path": "docker.container.image", - "type": "alias" - } - } - }, - "name": { - "path": "docker.container.name", - "type": "alias" - } - } - }, - "destination": { - "properties": { - "bytes": { - "type": "long" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "packets": { - "type": "long" - }, - "port": { - "type": "long" - } - } - }, - "docker": { - "properties": { - "container": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "ignore_above": 1024, - "type": "keyword" - }, - "labels": { - "type": "object" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "elasticsearch": { - "properties": { - "audit": { - "properties": { - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "indices": { - "ignore_above": 1024, - "type": "keyword" - }, - "layer": { - "ignore_above": 1024, - "type": "keyword" - }, - "origin_address": { - "type": "ip" - }, - "origin_port": { - "type": "long" - }, - "origin_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "principal": { - "ignore_above": 1024, - "type": "keyword" - }, - "realm": { - "ignore_above": 1024, - "type": "keyword" - }, - "request": { - "ignore_above": 1024, - "type": "keyword" - }, - "request_body": { - "norms": false, - "type": "text" - }, - "request_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "request_method": { - "ignore_above": 1024, - "type": "keyword" - }, - "roles": { - "ignore_above": 1024, - "type": "keyword" - }, - "uri": { - "ignore_above": 1024, - "type": "keyword" - }, - "uri_params": { - "norms": false, - "type": "text" - }, - "user_realm": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "deprecation": { - "properties": {} - }, - "gc": { - "properties": { - "heap": { - "properties": { - "size_kb": { - "type": "long" - }, - "used_kb": { - "type": "long" - } - } - }, - "jvm_runtime_sec": { - "type": "float" - }, - "old_gen": { - "properties": { - "size_kb": { - "type": "long" - }, - "used_kb": { - "type": "long" - } - } - }, - "phase": { - "properties": { - "class_unload_time_sec": { - "type": "float" - }, - "cpu_time": { - "properties": { - "real_sec": { - "type": "float" - }, - "sys_sec": { - "type": "float" - }, - "user_sec": { - "type": "float" - } - } - }, - "duration_sec": { - "type": "float" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "parallel_rescan_time_sec": { - "type": "float" - }, - "scrub_string_table_time_sec": { - "type": "float" - }, - "scrub_symbol_table_time_sec": { - "type": "float" - }, - "weak_refs_processing_time_sec": { - "type": "float" - } - } - }, - "stopping_threads_time_sec": { - "type": "float" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "threads_total_stop_time_sec": { - "type": "float" - }, - "young_gen": { - "properties": { - "size_kb": { - "type": "long" - }, - "used_kb": { - "type": "long" - } - } - } - } - }, - "index": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "node": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "server": { - "properties": { - "component": { - "ignore_above": 1024, - "type": "keyword" - }, - "gc": { - "properties": { - "collection_duration": { - "properties": { - "ms": { - "type": "float" - } - } - }, - "observation_duration": { - "properties": { - "ms": { - "type": "float" - } - } - }, - "overhead_seq": { - "type": "long" - }, - "young": { - "properties": { - "one": { - "type": "long" - }, - "two": { - "type": "long" - } - } - } - } - } - } - }, - "shard": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "slowlog": { - "properties": { - "extra_source": { - "norms": false, - "type": "text" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "logger": { - "ignore_above": 1024, - "type": "keyword" - }, - "routing": { - "ignore_above": 1024, - "type": "keyword" - }, - "search_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "source_query": { - "norms": false, - "type": "text" - }, - "stats": { - "norms": false, - "type": "text" - }, - "took": { - "norms": false, - "type": "text" - }, - "took_millis": { - "ignore_above": 1024, - "type": "keyword" - }, - "total_hits": { - "ignore_above": 1024, - "type": "keyword" - }, - "total_shards": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "types": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "error": { - "properties": { - "code": { - "type": "long" - }, - "message": { - "norms": false, - "type": "text" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "created": { - "type": "date" - }, - "dataset": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "type": "long" - }, - "end": { - "type": "date" - }, - "module": { - "path": "fileset.module", - "type": "alias" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - }, - "severity": { - "type": "long" - }, - "start": { - "type": "date" - }, - "timezone": { - "path": "beat.timezone", - "type": "alias" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "fields": { - "type": "object" - }, - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "size": { - "type": "long" - } - } - }, - "fileset": { - "properties": { - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "haproxy": { - "properties": { - "backend_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "backend_queue": { - "type": "long" - }, - "bind_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "bytes_read": { - "type": "long" - }, - "client": { - "properties": { - "ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - } - } - }, - "connection_wait_time_ms": { - "type": "long" - }, - "connections": { - "properties": { - "active": { - "type": "long" - }, - "backend": { - "type": "long" - }, - "frontend": { - "type": "long" - }, - "retries": { - "type": "long" - }, - "server": { - "type": "long" - } - } - }, - "destination": { - "properties": { - "ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - } - } - }, - "error_message": { - "norms": false, - "type": "text" - }, - "frontend_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http": { - "properties": { - "request": { - "properties": { - "captured_cookie": { - "ignore_above": 1024, - "type": "keyword" - }, - "captured_headers": { - "norms": false, - "type": "text" - }, - "raw_request_line": { - "norms": false, - "type": "text" - }, - "time_active_ms": { - "type": "long" - }, - "time_wait_ms": { - "type": "long" - }, - "time_wait_without_data_ms": { - "type": "long" - } - } - }, - "response": { - "properties": { - "captured_cookie": { - "ignore_above": 1024, - "type": "keyword" - }, - "captured_headers": { - "norms": false, - "type": "text" - }, - "status_code": { - "type": "long" - } - } - } - } - }, - "mode": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "process_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "server_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "server_queue": { - "type": "long" - }, - "source": { - "norms": false, - "type": "text" - }, - "tcp": { - "properties": { - "connection_waiting_time_ms": { - "type": "long" - }, - "processing_time_ms": { - "type": "long" - } - } - }, - "termination_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "time_backend_connect": { - "type": "long" - }, - "time_queue": { - "type": "long" - }, - "total_waiting_time_ms": { - "type": "long" - } - } - }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "containerized": { - "type": "boolean" - }, - "hostname": { - "path": "beat.hostname", - "type": "alias" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "http": { - "properties": { - "request": { - "properties": { - "method": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "response": { - "properties": { - "body": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "content_length": { - "type": "long" - }, - "elapsed_time": { - "type": "long" - }, - "status_code": { - "type": "long" - } - } - } - } - }, - "icinga": { - "properties": { - "debug": { - "properties": { - "facility": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "severity": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "main": { - "properties": { - "facility": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "severity": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "startup": { - "properties": { - "facility": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "severity": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "iis": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_received": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "cookie": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - }, - "query_string": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "request_time_ms": { - "type": "long" - }, - "response_code": { - "type": "long" - }, - "server_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "server_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "site_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "sub_status": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "index": false, - "norms": false, - "type": "text" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "win32_status": { - "type": "long" - } - } - }, - "error": { - "properties": { - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "queue_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "reason_phrase": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_port": { - "type": "long" - }, - "response_code": { - "type": "long" - }, - "server_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "server_port": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "input": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "iptables": { - "properties": { - "ether_type": { - "type": "long" - }, - "flow_label": { - "type": "long" - }, - "fragment_flags": { - "ignore_above": 1024, - "type": "keyword" - }, - "fragment_offset": { - "type": "long" - }, - "icmp": { - "properties": { - "code": { - "type": "long" - }, - "id": { - "type": "long" - }, - "parameter": { - "type": "long" - }, - "redirect": { - "type": "ip" - }, - "seq": { - "type": "long" - }, - "type": { - "type": "long" - } - } - }, - "id": { - "type": "long" - }, - "incomplete_bytes": { - "type": "long" - }, - "input_device": { - "ignore_above": 1024, - "type": "keyword" - }, - "length": { - "type": "long" - }, - "output_device": { - "ignore_above": 1024, - "type": "keyword" - }, - "precedence_bits": { - "type": "short" - }, - "tcp": { - "properties": { - "ack": { - "type": "long" - }, - "flags": { - "ignore_above": 1024, - "type": "keyword" - }, - "reserved_bits": { - "type": "short" - }, - "seq": { - "type": "long" - }, - "window": { - "type": "long" - } - } - }, - "tos": { - "type": "long" - }, - "ttl": { - "type": "long" - }, - "ubiquiti": { - "properties": { - "input_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "output_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "rule_number": { - "ignore_above": 1024, - "type": "keyword" - }, - "rule_set": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "udp": { - "properties": { - "length": { - "type": "long" - } - } - } - } - }, - "kafka": { - "properties": { - "log": { - "properties": { - "class": { - "norms": false, - "type": "text" - }, - "component": { - "ignore_above": 1024, - "type": "keyword" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "trace": { - "properties": { - "class": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "norms": false, - "type": "text" - }, - "message": { - "norms": false, - "type": "text" - } - } - } - } - } - } - }, - "kibana": { - "properties": { - "log": { - "properties": { - "meta": { - "type": "object" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "kubernetes": { - "properties": { - "annotations": { - "type": "object" - }, - "container": { - "properties": { - "image": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "labels": { - "type": "object" - }, - "namespace": { - "ignore_above": 1024, - "type": "keyword" - }, - "node": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pod": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "uid": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "log": { - "properties": { - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "flags": { - "ignore_above": 1024, - "type": "keyword" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "doc_values": false, - "ignore_above": 1024, - "index": false, - "type": "keyword" - }, - "source": { - "properties": { - "address": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "logstash": { - "properties": { - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "log_event": { - "type": "object" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "thread": { - "norms": false, - "type": "text" - } - } - }, - "slowlog": { - "properties": { - "event": { - "norms": false, - "type": "text" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "plugin_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "plugin_params": { - "norms": false, - "type": "text" - }, - "plugin_params_object": { - "type": "object" - }, - "plugin_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "thread": { - "norms": false, - "type": "text" - }, - "took_in_millis": { - "type": "long" - }, - "took_in_nanos": { - "type": "long" - } - } - } - } - }, - "message": { - "norms": false, - "type": "text" - }, - "meta": { - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "instance_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "instance_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "project_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "mongodb": { - "properties": { - "log": { - "properties": { - "component": { - "ignore_above": 1024, - "type": "keyword" - }, - "context": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "severity": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "mysql": { - "properties": { - "error": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "thread_id": { - "type": "long" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "slowlog": { - "properties": { - "bytes_sent": { - "type": "long" - }, - "current_user": { - "ignore_above": 1024, - "type": "keyword" - }, - "filesort": { - "type": "boolean" - }, - "filesort_on_disk": { - "type": "boolean" - }, - "full_join": { - "type": "boolean" - }, - "full_scan": { - "type": "boolean" - }, - "host": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "type": "long" - }, - "innodb": { - "properties": { - "io_r_bytes": { - "type": "long" - }, - "io_r_ops": { - "type": "long" - }, - "io_r_wait": { - "properties": { - "sec": { - "type": "long" - } - } - }, - "pages_distinct": { - "type": "long" - }, - "queue_wait": { - "properties": { - "sec": { - "type": "long" - } - } - }, - "rec_lock_wait": { - "properties": { - "sec": { - "type": "long" - } - } - }, - "trx_id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "killed": { - "ignore_above": 1024, - "type": "keyword" - }, - "last_errno": { - "ignore_above": 1024, - "type": "keyword" - }, - "lock_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "log_slow_rate_limit": { - "ignore_above": 1024, - "type": "keyword" - }, - "log_slow_rate_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "merge_passes": { - "type": "long" - }, - "priority_queue": { - "type": "boolean" - }, - "query": { - "ignore_above": 1024, - "type": "keyword" - }, - "query_cache_hit": { - "type": "boolean" - }, - "query_time": { - "properties": { - "sec": { - "type": "float" - } - } - }, - "rows_affected": { - "type": "long" - }, - "rows_examined": { - "type": "long" - }, - "rows_sent": { - "type": "long" - }, - "schema": { - "ignore_above": 1024, - "type": "keyword" - }, - "timestamp": { - "type": "long" - }, - "tmp_disk_tables": { - "type": "long" - }, - "tmp_table": { - "type": "boolean" - }, - "tmp_table_on_disk": { - "type": "boolean" - }, - "tmp_table_sizes": { - "type": "long" - }, - "tmp_tables": { - "type": "long" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "netflow": { - "properties": { - "absolute_error": { - "type": "double" - }, - "address_pool_high_threshold": { - "type": "long" - }, - "address_pool_low_threshold": { - "type": "long" - }, - "address_port_mapping_high_threshold": { - "type": "long" - }, - "address_port_mapping_low_threshold": { - "type": "long" - }, - "address_port_mapping_per_user_high_threshold": { - "type": "long" - }, - "anonymization_flags": { - "type": "long" - }, - "anonymization_technique": { - "type": "long" - }, - "application_category_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "application_description": { - "ignore_above": 1024, - "type": "keyword" - }, - "application_group_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "application_id": { - "type": "short" - }, - "application_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "application_sub_category_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "bgp_destination_as_number": { - "type": "long" - }, - "bgp_next_adjacent_as_number": { - "type": "long" - }, - "bgp_next_hop_ipv4_address": { - "type": "ip" - }, - "bgp_next_hop_ipv6_address": { - "type": "ip" - }, - "bgp_prev_adjacent_as_number": { - "type": "long" - }, - "bgp_source_as_number": { - "type": "long" - }, - "bgp_validity_state": { - "type": "short" - }, - "biflow_direction": { - "type": "short" - }, - "class_id": { - "type": "short" - }, - "class_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "classification_engine_id": { - "type": "short" - }, - "collection_time_milliseconds": { - "type": "date" - }, - "collector_certificate": { - "type": "short" - }, - "collector_ipv4_address": { - "type": "ip" - }, - "collector_ipv6_address": { - "type": "ip" - }, - "collector_transport_port": { - "type": "long" - }, - "common_properties_id": { - "type": "long" - }, - "confidence_level": { - "type": "double" - }, - "connection_sum_duration_seconds": { - "type": "long" - }, - "connection_transaction_id": { - "type": "long" - }, - "data_link_frame_section": { - "type": "short" - }, - "data_link_frame_size": { - "type": "long" - }, - "data_link_frame_type": { - "type": "long" - }, - "data_records_reliability": { - "type": "boolean" - }, - "delta_flow_count": { - "type": "long" - }, - "destination_ipv4_address": { - "type": "ip" - }, - "destination_ipv4_prefix": { - "type": "ip" - }, - "destination_ipv4_prefix_length": { - "type": "short" - }, - "destination_ipv6_address": { - "type": "ip" - }, - "destination_ipv6_prefix": { - "type": "ip" - }, - "destination_ipv6_prefix_length": { - "type": "short" - }, - "destination_mac_address": { - "ignore_above": 1024, - "type": "keyword" - }, - "destination_transport_port": { - "type": "long" - }, - "digest_hash_value": { - "type": "long" - }, - "distinct_count_of_destinatio_nipa_ddress": { - "type": "long" - }, - "distinct_count_of_destination_ipv4_address": { - "type": "long" - }, - "distinct_count_of_destination_ipv6_address": { - "type": "long" - }, - "distinct_count_of_sourc_eipa_ddress": { - "type": "long" - }, - "distinct_count_of_source_ipv4_address": { - "type": "long" - }, - "distinct_count_of_source_ipv6_address": { - "type": "long" - }, - "dot1q_customer_dei": { - "type": "boolean" - }, - "dot1q_customer_destination_mac_address": { - "ignore_above": 1024, - "type": "keyword" - }, - "dot1q_customer_priority": { - "type": "short" - }, - "dot1q_customer_source_mac_address": { - "ignore_above": 1024, - "type": "keyword" - }, - "dot1q_customer_vlan_id": { - "type": "long" - }, - "dot1q_dei": { - "type": "boolean" - }, - "dot1q_priority": { - "type": "short" - }, - "dot1q_service_instance_id": { - "type": "long" - }, - "dot1q_service_instance_priority": { - "type": "short" - }, - "dot1q_service_instance_tag": { - "type": "short" - }, - "dot1q_vlan_id": { - "type": "long" - }, - "dropped_layer2_octet_delta_count": { - "type": "long" - }, - "dropped_layer2_octet_total_count": { - "type": "long" - }, - "dropped_octet_delta_count": { - "type": "long" - }, - "dropped_octet_total_count": { - "type": "long" - }, - "dropped_packet_delta_count": { - "type": "long" - }, - "dropped_packet_total_count": { - "type": "long" - }, - "dst_traffic_index": { - "type": "long" - }, - "egress_broadcast_packet_total_count": { - "type": "long" - }, - "egress_interface": { - "type": "long" - }, - "egress_interface_type": { - "type": "long" - }, - "egress_physical_interface": { - "type": "long" - }, - "egress_unicast_packet_total_count": { - "type": "long" - }, - "egress_vrfid": { - "type": "long" - }, - "encrypted_technology": { - "ignore_above": 1024, - "type": "keyword" - }, - "engine_id": { - "type": "short" - }, - "engine_type": { - "type": "short" - }, - "ethernet_header_length": { - "type": "short" - }, - "ethernet_payload_length": { - "type": "long" - }, - "ethernet_total_length": { - "type": "long" - }, - "ethernet_type": { - "type": "long" - }, - "export_interface": { - "type": "long" - }, - "export_protocol_version": { - "type": "short" - }, - "export_sctp_stream_id": { - "type": "long" - }, - "export_transport_protocol": { - "type": "short" - }, - "exported_flow_record_total_count": { - "type": "long" - }, - "exported_message_total_count": { - "type": "long" - }, - "exported_octet_total_count": { - "type": "long" - }, - "exporter": { - "properties": { - "address": { - "ignore_above": 1024, - "type": "keyword" - }, - "source_id": { - "type": "long" - }, - "timestamp": { - "type": "date" - }, - "uptime_millis": { - "type": "long" - }, - "version": { - "type": "long" - } - } - }, - "exporter_certificate": { - "type": "short" - }, - "exporter_ipv4_address": { - "type": "ip" - }, - "exporter_ipv6_address": { - "type": "ip" - }, - "exporter_transport_port": { - "type": "long" - }, - "exporting_process_id": { - "type": "long" - }, - "external_address_realm": { - "type": "short" - }, - "firewall_event": { - "type": "short" - }, - "flags_and_sampler_id": { - "type": "long" - }, - "flow_active_timeout": { - "type": "long" - }, - "flow_direction": { - "type": "short" - }, - "flow_duration_microseconds": { - "type": "long" - }, - "flow_duration_milliseconds": { - "type": "long" - }, - "flow_end_delta_microseconds": { - "type": "long" - }, - "flow_end_microseconds": { - "type": "date" - }, - "flow_end_milliseconds": { - "type": "date" - }, - "flow_end_nanoseconds": { - "type": "date" - }, - "flow_end_reason": { - "type": "short" - }, - "flow_end_seconds": { - "type": "date" - }, - "flow_end_sys_up_time": { - "type": "long" - }, - "flow_id": { - "type": "long" - }, - "flow_idle_timeout": { - "type": "long" - }, - "flow_key_indicator": { - "type": "long" - }, - "flow_label_ipv6": { - "type": "long" - }, - "flow_sampling_time_interval": { - "type": "long" - }, - "flow_sampling_time_spacing": { - "type": "long" - }, - "flow_selected_flow_delta_count": { - "type": "long" - }, - "flow_selected_octet_delta_count": { - "type": "long" - }, - "flow_selected_packet_delta_count": { - "type": "long" - }, - "flow_selector_algorithm": { - "type": "long" - }, - "flow_start_delta_microseconds": { - "type": "long" - }, - "flow_start_microseconds": { - "type": "date" - }, - "flow_start_milliseconds": { - "type": "date" - }, - "flow_start_nanoseconds": { - "type": "date" - }, - "flow_start_seconds": { - "type": "date" - }, - "flow_start_sys_up_time": { - "type": "long" - }, - "forwarding_status": { - "type": "short" - }, - "fragment_flags": { - "type": "short" - }, - "fragment_identification": { - "type": "long" - }, - "fragment_offset": { - "type": "long" - }, - "global_address_mapping_high_threshold": { - "type": "long" - }, - "gre_key": { - "type": "long" - }, - "hash_digest_output": { - "type": "boolean" - }, - "hash_flow_domain": { - "type": "long" - }, - "hash_initialiser_value": { - "type": "long" - }, - "hash_ipp_ayload_offset": { - "type": "long" - }, - "hash_ipp_ayload_size": { - "type": "long" - }, - "hash_output_range_max": { - "type": "long" - }, - "hash_output_range_min": { - "type": "long" - }, - "hash_selected_range_max": { - "type": "long" - }, - "hash_selected_range_min": { - "type": "long" - }, - "http_content_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_message_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_reason_phrase": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_request_host": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_request_method": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_request_target": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_status_code": { - "type": "long" - }, - "http_user_agent": { - "ignore_above": 1024, - "type": "keyword" - }, - "icmp_code_ipv4": { - "type": "short" - }, - "icmp_code_ipv6": { - "type": "short" - }, - "icmp_type_code_ipv4": { - "type": "long" - }, - "icmp_type_code_ipv6": { - "type": "long" - }, - "icmp_type_ipv4": { - "type": "short" - }, - "icmp_type_ipv6": { - "type": "short" - }, - "igmp_type": { - "type": "short" - }, - "ignored_data_record_total_count": { - "type": "long" - }, - "ignored_layer2_frame_total_count": { - "type": "long" - }, - "ignored_layer2_octet_total_count": { - "type": "long" - }, - "ignored_octet_total_count": { - "type": "long" - }, - "ignored_packet_total_count": { - "type": "long" - }, - "information_element_data_type": { - "type": "short" - }, - "information_element_description": { - "ignore_above": 1024, - "type": "keyword" - }, - "information_element_id": { - "type": "long" - }, - "information_element_index": { - "type": "long" - }, - "information_element_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "information_element_range_begin": { - "type": "long" - }, - "information_element_range_end": { - "type": "long" - }, - "information_element_semantics": { - "type": "short" - }, - "information_element_units": { - "type": "long" - }, - "ingress_broadcast_packet_total_count": { - "type": "long" - }, - "ingress_interface": { - "type": "long" - }, - "ingress_interface_type": { - "type": "long" - }, - "ingress_multicast_packet_total_count": { - "type": "long" - }, - "ingress_physical_interface": { - "type": "long" - }, - "ingress_unicast_packet_total_count": { - "type": "long" - }, - "ingress_vrfid": { - "type": "long" - }, - "initiator_octets": { - "type": "long" - }, - "initiator_packets": { - "type": "long" - }, - "interface_description": { - "ignore_above": 1024, - "type": "keyword" - }, - "interface_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "intermediate_process_id": { - "type": "long" - }, - "internal_address_realm": { - "type": "short" - }, - "ip_class_of_service": { - "type": "short" - }, - "ip_diff_serv_code_point": { - "type": "short" - }, - "ip_header_length": { - "type": "short" - }, - "ip_header_packet_section": { - "type": "short" - }, - "ip_next_hop_ipv4_address": { - "type": "ip" - }, - "ip_next_hop_ipv6_address": { - "type": "ip" - }, - "ip_payload_length": { - "type": "long" - }, - "ip_payload_packet_section": { - "type": "short" - }, - "ip_precedence": { - "type": "short" - }, - "ip_sec_spi": { - "type": "long" - }, - "ip_total_length": { - "type": "long" - }, - "ip_ttl": { - "type": "short" - }, - "ip_version": { - "type": "short" - }, - "ipv4_ihl": { - "type": "short" - }, - "ipv4_options": { - "type": "long" - }, - "ipv4_router_sc": { - "type": "ip" - }, - "ipv6_extension_headers": { - "type": "long" - }, - "is_multicast": { - "type": "short" - }, - "layer2_frame_delta_count": { - "type": "long" - }, - "layer2_frame_total_count": { - "type": "long" - }, - "layer2_octet_delta_count": { - "type": "long" - }, - "layer2_octet_delta_sum_of_squares": { - "type": "long" - }, - "layer2_octet_total_count": { - "type": "long" - }, - "layer2_octet_total_sum_of_squares": { - "type": "long" - }, - "layer2_segment_id": { - "type": "long" - }, - "layer2packet_section_data": { - "type": "short" - }, - "layer2packet_section_offset": { - "type": "long" - }, - "layer2packet_section_size": { - "type": "long" - }, - "line_card_id": { - "type": "long" - }, - "lower_cli_imit": { - "type": "double" - }, - "max_bieb_ntries": { - "type": "long" - }, - "max_entries_per_user": { - "type": "long" - }, - "max_export_seconds": { - "type": "date" - }, - "max_flow_end_microseconds": { - "type": "date" - }, - "max_flow_end_milliseconds": { - "type": "date" - }, - "max_flow_end_nanoseconds": { - "type": "date" - }, - "max_flow_end_seconds": { - "type": "date" - }, - "max_fragments_pending_reassembly": { - "type": "long" - }, - "max_session_entries": { - "type": "long" - }, - "max_subscribers": { - "type": "long" - }, - "maximum_ip_total_length": { - "type": "long" - }, - "maximum_layer2_total_length": { - "type": "long" - }, - "maximum_ttl": { - "type": "short" - }, - "message_md5_checksum": { - "type": "short" - }, - "message_scope": { - "type": "short" - }, - "metering_process_id": { - "type": "long" - }, - "metro_evc_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "metro_evc_type": { - "type": "short" - }, - "mib_capture_time_semantics": { - "type": "short" - }, - "mib_context_engine_id": { - "type": "short" - }, - "mib_context_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "mib_index_indicator": { - "type": "long" - }, - "mib_module_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "mib_object_description": { - "ignore_above": 1024, - "type": "keyword" - }, - "mib_object_identifier": { - "type": "short" - }, - "mib_object_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "mib_object_syntax": { - "ignore_above": 1024, - "type": "keyword" - }, - "mib_object_value_bits": { - "type": "short" - }, - "mib_object_value_counter": { - "type": "long" - }, - "mib_object_value_gauge": { - "type": "long" - }, - "mib_object_value_integer": { - "type": "long" - }, - "mib_object_value_octet_string": { - "type": "short" - }, - "mib_object_value_oid": { - "type": "short" - }, - "mib_object_value_time_ticks": { - "type": "long" - }, - "mib_object_value_unsigned": { - "type": "long" - }, - "mib_object_valuei_pa_ddress": { - "type": "ip" - }, - "mib_sub_identifier": { - "type": "long" - }, - "min_export_seconds": { - "type": "date" - }, - "min_flow_start_microseconds": { - "type": "date" - }, - "min_flow_start_milliseconds": { - "type": "date" - }, - "min_flow_start_nanoseconds": { - "type": "date" - }, - "min_flow_start_seconds": { - "type": "date" - }, - "minimum_ip_total_length": { - "type": "long" - }, - "minimum_layer2_total_length": { - "type": "long" - }, - "minimum_ttl": { - "type": "short" - }, - "mobile_imsi": { - "ignore_above": 1024, - "type": "keyword" - }, - "mobile_msisdn": { - "ignore_above": 1024, - "type": "keyword" - }, - "monitoring_interval_end_milli_seconds": { - "type": "date" - }, - "monitoring_interval_start_milli_seconds": { - "type": "date" - }, - "mpls_label_stack_depth": { - "type": "long" - }, - "mpls_label_stack_length": { - "type": "long" - }, - "mpls_label_stack_section": { - "type": "short" - }, - "mpls_label_stack_section10": { - "type": "short" - }, - "mpls_label_stack_section2": { - "type": "short" - }, - "mpls_label_stack_section3": { - "type": "short" - }, - "mpls_label_stack_section4": { - "type": "short" - }, - "mpls_label_stack_section5": { - "type": "short" - }, - "mpls_label_stack_section6": { - "type": "short" - }, - "mpls_label_stack_section7": { - "type": "short" - }, - "mpls_label_stack_section8": { - "type": "short" - }, - "mpls_label_stack_section9": { - "type": "short" - }, - "mpls_payload_length": { - "type": "long" - }, - "mpls_payload_packet_section": { - "type": "short" - }, - "mpls_top_label_exp": { - "type": "short" - }, - "mpls_top_label_ipv4_address": { - "type": "ip" - }, - "mpls_top_label_ipv6_address": { - "type": "ip" - }, - "mpls_top_label_prefix_length": { - "type": "short" - }, - "mpls_top_label_stack_section": { - "type": "short" - }, - "mpls_top_label_ttl": { - "type": "short" - }, - "mpls_top_label_type": { - "type": "short" - }, - "mpls_vpn_route_distinguisher": { - "type": "short" - }, - "multicast_replication_factor": { - "type": "long" - }, - "nat_event": { - "type": "short" - }, - "nat_instance_id": { - "type": "long" - }, - "nat_originating_address_realm": { - "type": "short" - }, - "nat_pool_id": { - "type": "long" - }, - "nat_pool_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "nat_quota_exceeded_event": { - "type": "long" - }, - "nat_threshold_event": { - "type": "long" - }, - "nat_type": { - "type": "short" - }, - "new_connection_delta_count": { - "type": "long" - }, - "next_header_ipv6": { - "type": "short" - }, - "not_sent_flow_total_count": { - "type": "long" - }, - "not_sent_layer2_octet_total_count": { - "type": "long" - }, - "not_sent_octet_total_count": { - "type": "long" - }, - "not_sent_packet_total_count": { - "type": "long" - }, - "observation_domain_id": { - "type": "long" - }, - "observation_domain_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "observation_point_id": { - "type": "long" - }, - "observation_point_type": { - "type": "short" - }, - "observation_time_microseconds": { - "type": "date" - }, - "observation_time_milliseconds": { - "type": "date" - }, - "observation_time_nanoseconds": { - "type": "date" - }, - "observation_time_seconds": { - "type": "date" - }, - "observed_flow_total_count": { - "type": "long" - }, - "octet_delta_count": { - "type": "long" - }, - "octet_delta_sum_of_squares": { - "type": "long" - }, - "octet_total_count": { - "type": "long" - }, - "octet_total_sum_of_squares": { - "type": "long" - }, - "opaque_octets": { - "type": "short" - }, - "original_exporter_ipv4_address": { - "type": "ip" - }, - "original_exporter_ipv6_address": { - "type": "ip" - }, - "original_flows_completed": { - "type": "long" - }, - "original_flows_initiated": { - "type": "long" - }, - "original_flows_present": { - "type": "long" - }, - "original_observation_domain_id": { - "type": "long" - }, - "p2p_technology": { - "ignore_above": 1024, - "type": "keyword" - }, - "packet_delta_count": { - "type": "long" - }, - "packet_total_count": { - "type": "long" - }, - "padding_octets": { - "type": "short" - }, - "payload_length_ipv6": { - "type": "long" - }, - "port_id": { - "type": "long" - }, - "port_range_end": { - "type": "long" - }, - "port_range_num_ports": { - "type": "long" - }, - "port_range_start": { - "type": "long" - }, - "port_range_step_size": { - "type": "long" - }, - "post_destination_mac_address": { - "ignore_above": 1024, - "type": "keyword" - }, - "post_dot1q_customer_vlan_id": { - "type": "long" - }, - "post_dot1q_vlan_id": { - "type": "long" - }, - "post_ip_class_of_service": { - "type": "short" - }, - "post_ip_diff_serv_code_point": { - "type": "short" - }, - "post_ip_precedence": { - "type": "short" - }, - "post_layer2_octet_delta_count": { - "type": "long" - }, - "post_layer2_octet_total_count": { - "type": "long" - }, - "post_mcast_layer2_octet_delta_count": { - "type": "long" - }, - "post_mcast_layer2_octet_total_count": { - "type": "long" - }, - "post_mcast_octet_delta_count": { - "type": "long" - }, - "post_mcast_octet_total_count": { - "type": "long" - }, - "post_mcast_packet_delta_count": { - "type": "long" - }, - "post_mcast_packet_total_count": { - "type": "long" - }, - "post_mpls_top_label_exp": { - "type": "short" - }, - "post_nadt_estination_ipv4_address": { - "type": "ip" - }, - "post_nadt_estination_ipv6_address": { - "type": "ip" - }, - "post_napdt_estination_transport_port": { - "type": "long" - }, - "post_napst_ource_transport_port": { - "type": "long" - }, - "post_nast_ource_ipv4_address": { - "type": "ip" - }, - "post_nast_ource_ipv6_address": { - "type": "ip" - }, - "post_octet_delta_count": { - "type": "long" - }, - "post_octet_total_count": { - "type": "long" - }, - "post_packet_delta_count": { - "type": "long" - }, - "post_packet_total_count": { - "type": "long" - }, - "post_source_mac_address": { - "ignore_above": 1024, - "type": "keyword" - }, - "post_vlan_id": { - "type": "long" - }, - "private_enterprise_number": { - "type": "long" - }, - "protocol_identifier": { - "type": "short" - }, - "pseudo_wire_control_word": { - "type": "long" - }, - "pseudo_wire_destination_ipv4_address": { - "type": "ip" - }, - "pseudo_wire_id": { - "type": "long" - }, - "pseudo_wire_type": { - "type": "long" - }, - "relative_error": { - "type": "double" - }, - "responder_octets": { - "type": "long" - }, - "responder_packets": { - "type": "long" - }, - "rfc3550_jitter_microseconds": { - "type": "long" - }, - "rfc3550_jitter_milliseconds": { - "type": "long" - }, - "rfc3550_jitter_nanoseconds": { - "type": "long" - }, - "rtp_sequence_number": { - "type": "long" - }, - "sampler_id": { - "type": "short" - }, - "sampler_mode": { - "type": "short" - }, - "sampler_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "sampler_random_interval": { - "type": "long" - }, - "sampling_algorithm": { - "type": "short" - }, - "sampling_flow_interval": { - "type": "long" - }, - "sampling_flow_spacing": { - "type": "long" - }, - "sampling_interval": { - "type": "long" - }, - "sampling_packet_interval": { - "type": "long" - }, - "sampling_packet_space": { - "type": "long" - }, - "sampling_population": { - "type": "long" - }, - "sampling_probability": { - "type": "double" - }, - "sampling_size": { - "type": "long" - }, - "sampling_time_interval": { - "type": "long" - }, - "sampling_time_space": { - "type": "long" - }, - "section_exported_octets": { - "type": "long" - }, - "section_offset": { - "type": "long" - }, - "selection_sequence_id": { - "type": "long" - }, - "selector_algorithm": { - "type": "long" - }, - "selector_id": { - "type": "long" - }, - "selector_id_total_pkts_observed": { - "type": "long" - }, - "selector_id_total_pkts_selected": { - "type": "long" - }, - "selector_itd_otal_flows_observed": { - "type": "long" - }, - "selector_itd_otal_flows_selected": { - "type": "long" - }, - "selector_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "session_scope": { - "type": "short" - }, - "source_ipv4_address": { - "type": "ip" - }, - "source_ipv4_prefix": { - "type": "ip" - }, - "source_ipv4_prefix_length": { - "type": "short" - }, - "source_ipv6_address": { - "type": "ip" - }, - "source_ipv6_prefix": { - "type": "ip" - }, - "source_ipv6_prefix_length": { - "type": "short" - }, - "source_mac_address": { - "ignore_above": 1024, - "type": "keyword" - }, - "source_transport_port": { - "type": "long" - }, - "source_transport_ports_limit": { - "type": "long" - }, - "src_traffic_index": { - "type": "long" - }, - "sta_ipv4_address": { - "type": "ip" - }, - "sta_mac_address": { - "ignore_above": 1024, - "type": "keyword" - }, - "system_init_time_milliseconds": { - "type": "date" - }, - "tcp_ack_total_count": { - "type": "long" - }, - "tcp_acknowledgement_number": { - "type": "long" - }, - "tcp_control_bits": { - "type": "long" - }, - "tcp_destination_port": { - "type": "long" - }, - "tcp_fin_total_count": { - "type": "long" - }, - "tcp_header_length": { - "type": "short" - }, - "tcp_options": { - "type": "long" - }, - "tcp_psh_total_count": { - "type": "long" - }, - "tcp_rst_total_count": { - "type": "long" - }, - "tcp_sequence_number": { - "type": "long" - }, - "tcp_source_port": { - "type": "long" - }, - "tcp_syn_total_count": { - "type": "long" - }, - "tcp_urg_total_count": { - "type": "long" - }, - "tcp_urgent_pointer": { - "type": "long" - }, - "tcp_window_scale": { - "type": "long" - }, - "tcp_window_size": { - "type": "long" - }, - "template_id": { - "type": "long" - }, - "total_length_ipv4": { - "type": "long" - }, - "transport_octet_delta_count": { - "type": "long" - }, - "transport_packet_delta_count": { - "type": "long" - }, - "tunnel_technology": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "udp_destination_port": { - "type": "long" - }, - "udp_message_length": { - "type": "long" - }, - "udp_source_port": { - "type": "long" - }, - "upper_cli_imit": { - "type": "double" - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "value_distribution_method": { - "type": "short" - }, - "virtual_station_interface_id": { - "type": "short" - }, - "virtual_station_interface_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "virtual_station_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "virtual_station_uuid": { - "type": "short" - }, - "vlan_id": { - "type": "long" - }, - "vpn_identifier": { - "type": "short" - }, - "vr_fname": { - "ignore_above": 1024, - "type": "keyword" - }, - "wlan_channel_id": { - "type": "short" - }, - "wlan_ssid": { - "ignore_above": 1024, - "type": "keyword" - }, - "wtp_mac_address": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "network": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - }, - "protocol": { - "ignore_above": 1024, - "type": "keyword" - }, - "transport": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "nginx": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "index": false, - "norms": false, - "type": "text" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "connection_id": { - "type": "long" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "tid": { - "type": "long" - } - } - } - } - }, - "offset": { - "type": "long" - }, - "osquery": { - "properties": { - "result": { - "properties": { - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "calendar_time": { - "ignore_above": 1024, - "type": "keyword" - }, - "host_identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "unix_time": { - "type": "long" - } - } - } - } - }, - "postgresql": { - "properties": { - "log": { - "properties": { - "core_id": { - "type": "long" - }, - "database": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "type": "float" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "query": { - "ignore_above": 1024, - "type": "keyword" - }, - "thread_id": { - "type": "long" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "prospector": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "read_timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "redis": { - "properties": { - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "role": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "slowlog": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "cmd": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "properties": { - "us": { - "type": "long" - } - } - }, - "id": { - "type": "long" - }, - "key": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "service": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "source": { - "ignore_above": 1024, - "type": "keyword" - }, - "source_ecs": { - "properties": { - "bytes": { - "type": "long" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "packets": { - "type": "long" - }, - "port": { - "type": "long" - } - } - }, - "stream": { - "ignore_above": 1024, - "type": "keyword" - }, - "suricata": { - "properties": { - "eve": { - "properties": { - "alert": { - "properties": { - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "gid": { - "type": "long" - }, - "rev": { - "type": "long" - }, - "severity": { - "type": "long" - }, - "signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "signature_id": { - "type": "long" - } - } - }, - "app_proto": { - "ignore_above": 1024, - "type": "keyword" - }, - "app_proto_expected": { - "ignore_above": 1024, - "type": "keyword" - }, - "app_proto_orig": { - "ignore_above": 1024, - "type": "keyword" - }, - "app_proto_tc": { - "ignore_above": 1024, - "type": "keyword" - }, - "app_proto_ts": { - "ignore_above": 1024, - "type": "keyword" - }, - "dest_ip": { - "type": "ip" - }, - "dest_port": { - "type": "long" - }, - "dns": { - "properties": { - "id": { - "type": "long" - }, - "rcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "rdata": { - "ignore_above": 1024, - "type": "keyword" - }, - "rrname": { - "ignore_above": 1024, - "type": "keyword" - }, - "rrtype": { - "ignore_above": 1024, - "type": "keyword" - }, - "ttl": { - "type": "long" - }, - "tx_id": { - "type": "long" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "email": { - "properties": { - "status": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "fileinfo": { - "properties": { - "filename": { - "ignore_above": 1024, - "type": "keyword" - }, - "gaps": { - "type": "boolean" - }, - "md5": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "size": { - "type": "long" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "stored": { - "type": "boolean" - }, - "tx_id": { - "type": "long" - } - } - }, - "flags": { - "properties": {} - }, - "flow": { - "properties": { - "age": { - "type": "long" - }, - "alerted": { - "type": "boolean" - }, - "bytes_toclient": { - "type": "long" - }, - "bytes_toserver": { - "type": "long" - }, - "end": { - "type": "date" - }, - "pkts_toclient": { - "type": "long" - }, - "pkts_toserver": { - "type": "long" - }, - "reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "start": { - "type": "date" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "flow_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "http": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_content_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_method": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_refer": { - "ignore_above": 1024, - "type": "keyword" - }, - "http_user_agent": { - "ignore_above": 1024, - "type": "keyword" - }, - "length": { - "type": "long" - }, - "protocol": { - "ignore_above": 1024, - "type": "keyword" - }, - "redirect": { - "ignore_above": 1024, - "type": "keyword" - }, - "status": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "icmp_code": { - "type": "long" - }, - "icmp_type": { - "type": "long" - }, - "in_iface": { - "ignore_above": 1024, - "type": "keyword" - }, - "pcap_cnt": { - "type": "long" - }, - "proto": { - "ignore_above": 1024, - "type": "keyword" - }, - "smtp": { - "properties": { - "helo": { - "ignore_above": 1024, - "type": "keyword" - }, - "mail_from": { - "ignore_above": 1024, - "type": "keyword" - }, - "rcpt_to": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "src_ip": { - "type": "ip" - }, - "src_port": { - "type": "long" - }, - "ssh": { - "properties": { - "client": { - "properties": { - "proto_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "software_version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "server": { - "properties": { - "proto_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "software_version": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "stats": { - "properties": { - "app_layer": { - "properties": { - "flow": { - "properties": { - "dcerpc_tcp": { - "type": "long" - }, - "dcerpc_udp": { - "type": "long" - }, - "dns_tcp": { - "type": "long" - }, - "dns_udp": { - "type": "long" - }, - "failed_tcp": { - "type": "long" - }, - "failed_udp": { - "type": "long" - }, - "ftp": { - "type": "long" - }, - "http": { - "type": "long" - }, - "imap": { - "type": "long" - }, - "msn": { - "type": "long" - }, - "smb": { - "type": "long" - }, - "smtp": { - "type": "long" - }, - "ssh": { - "type": "long" - }, - "tls": { - "type": "long" - } - } - }, - "tx": { - "properties": { - "dcerpc_tcp": { - "type": "long" - }, - "dcerpc_udp": { - "type": "long" - }, - "dns_tcp": { - "type": "long" - }, - "dns_udp": { - "type": "long" - }, - "ftp": { - "type": "long" - }, - "http": { - "type": "long" - }, - "smb": { - "type": "long" - }, - "smtp": { - "type": "long" - }, - "ssh": { - "type": "long" - }, - "tls": { - "type": "long" - } - } - } - } - }, - "capture": { - "properties": { - "kernel_drops": { - "type": "long" - }, - "kernel_ifdrops": { - "type": "long" - }, - "kernel_packets": { - "type": "long" - } - } - }, - "decoder": { - "properties": { - "avg_pkt_size": { - "type": "long" - }, - "bytes": { - "type": "long" - }, - "dce": { - "properties": { - "pkt_too_small": { - "type": "long" - } - } - }, - "erspan": { - "type": "long" - }, - "ethernet": { - "type": "long" - }, - "gre": { - "type": "long" - }, - "icmpv4": { - "type": "long" - }, - "icmpv6": { - "type": "long" - }, - "ieee8021ah": { - "type": "long" - }, - "invalid": { - "type": "long" - }, - "ipraw": { - "properties": { - "invalid_ip_version": { - "type": "long" - } - } - }, - "ipv4": { - "type": "long" - }, - "ipv4_in_ipv6": { - "type": "long" - }, - "ipv6": { - "type": "long" - }, - "ipv6_in_ipv6": { - "type": "long" - }, - "ltnull": { - "properties": { - "pkt_too_small": { - "type": "long" - }, - "unsupported_type": { - "type": "long" - } - } - }, - "max_pkt_size": { - "type": "long" - }, - "mpls": { - "type": "long" - }, - "null": { - "type": "long" - }, - "pkts": { - "type": "long" - }, - "ppp": { - "type": "long" - }, - "pppoe": { - "type": "long" - }, - "raw": { - "type": "long" - }, - "sctp": { - "type": "long" - }, - "sll": { - "type": "long" - }, - "tcp": { - "type": "long" - }, - "teredo": { - "type": "long" - }, - "udp": { - "type": "long" - }, - "vlan": { - "type": "long" - }, - "vlan_qinq": { - "type": "long" - } - } - }, - "defrag": { - "properties": { - "ipv4": { - "properties": { - "fragments": { - "type": "long" - }, - "reassembled": { - "type": "long" - }, - "timeouts": { - "type": "long" - } - } - }, - "ipv6": { - "properties": { - "fragments": { - "type": "long" - }, - "reassembled": { - "type": "long" - }, - "timeouts": { - "type": "long" - } - } - }, - "max_frag_hits": { - "type": "long" - } - } - }, - "detect": { - "properties": { - "alert": { - "type": "long" - } - } - }, - "dns": { - "properties": { - "memcap_global": { - "type": "long" - }, - "memcap_state": { - "type": "long" - }, - "memuse": { - "type": "long" - } - } - }, - "file_store": { - "properties": { - "open_files": { - "type": "long" - } - } - }, - "flow": { - "properties": { - "emerg_mode_entered": { - "type": "long" - }, - "emerg_mode_over": { - "type": "long" - }, - "icmpv4": { - "type": "long" - }, - "icmpv6": { - "type": "long" - }, - "memcap": { - "type": "long" - }, - "memuse": { - "type": "long" - }, - "spare": { - "type": "long" - }, - "tcp": { - "type": "long" - }, - "tcp_reuse": { - "type": "long" - }, - "udp": { - "type": "long" - } - } - }, - "flow_mgr": { - "properties": { - "bypassed_pruned": { - "type": "long" - }, - "closed_pruned": { - "type": "long" - }, - "est_pruned": { - "type": "long" - }, - "flows_checked": { - "type": "long" - }, - "flows_notimeout": { - "type": "long" - }, - "flows_removed": { - "type": "long" - }, - "flows_timeout": { - "type": "long" - }, - "flows_timeout_inuse": { - "type": "long" - }, - "new_pruned": { - "type": "long" - }, - "rows_busy": { - "type": "long" - }, - "rows_checked": { - "type": "long" - }, - "rows_empty": { - "type": "long" - }, - "rows_maxlen": { - "type": "long" - }, - "rows_skipped": { - "type": "long" - } - } - }, - "http": { - "properties": { - "memcap": { - "type": "long" - }, - "memuse": { - "type": "long" - } - } - }, - "tcp": { - "properties": { - "insert_data_normal_fail": { - "type": "long" - }, - "insert_data_overlap_fail": { - "type": "long" - }, - "insert_list_fail": { - "type": "long" - }, - "invalid_checksum": { - "type": "long" - }, - "memuse": { - "type": "long" - }, - "no_flow": { - "type": "long" - }, - "overlap": { - "type": "long" - }, - "overlap_diff_data": { - "type": "long" - }, - "pseudo": { - "type": "long" - }, - "pseudo_failed": { - "type": "long" - }, - "reassembly_gap": { - "type": "long" - }, - "reassembly_memuse": { - "type": "long" - }, - "rst": { - "type": "long" - }, - "segment_memcap_drop": { - "type": "long" - }, - "sessions": { - "type": "long" - }, - "ssn_memcap_drop": { - "type": "long" - }, - "stream_depth_reached": { - "type": "long" - }, - "syn": { - "type": "long" - }, - "synack": { - "type": "long" - } - } - }, - "uptime": { - "type": "long" - } - } - }, - "tcp": { - "properties": { - "ack": { - "type": "boolean" - }, - "fin": { - "type": "boolean" - }, - "psh": { - "type": "boolean" - }, - "rst": { - "type": "boolean" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "syn": { - "type": "boolean" - }, - "tcp_flags": { - "ignore_above": 1024, - "type": "keyword" - }, - "tcp_flags_tc": { - "ignore_above": 1024, - "type": "keyword" - }, - "tcp_flags_ts": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "timestamp": { - "type": "date" - }, - "tls": { - "properties": { - "fingerprint": { - "ignore_above": 1024, - "type": "keyword" - }, - "issuerdn": { - "ignore_above": 1024, - "type": "keyword" - }, - "notafter": { - "type": "date" - }, - "notbefore": { - "type": "date" - }, - "serial": { - "ignore_above": 1024, - "type": "keyword" - }, - "session_resumed": { - "type": "boolean" - }, - "sni": { - "ignore_above": 1024, - "type": "keyword" - }, - "subject": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "tx_id": { - "type": "long" - } - } - } - } - }, - "syslog": { - "properties": { - "facility": { - "type": "long" - }, - "facility_label": { - "ignore_above": 1024, - "type": "keyword" - }, - "priority": { - "type": "long" - }, - "severity_label": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "system": { - "properties": { - "auth": { - "properties": { - "groupadd": { - "properties": { - "gid": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "type": "long" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "ssh": { - "properties": { - "dropped_ip": { - "type": "ip" - }, - "event": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - }, - "signature": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "sudo": { - "properties": { - "command": { - "ignore_above": 1024, - "type": "keyword" - }, - "error": { - "ignore_above": 1024, - "type": "keyword" - }, - "pwd": { - "ignore_above": 1024, - "type": "keyword" - }, - "tty": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "useradd": { - "properties": { - "gid": { - "type": "long" - }, - "home": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "shell": { - "ignore_above": 1024, - "type": "keyword" - }, - "uid": { - "type": "long" - } - } - } - } - }, - "syslog": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "program": { - "ignore_above": 1024, - "type": "keyword" - }, - "timestamp": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "traefik": { - "properties": { - "access": { - "properties": { - "agent": { - "norms": false, - "type": "text" - }, - "backend_url": { - "norms": false, - "type": "text" - }, - "body_sent": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "duration": { - "type": "long" - }, - "frontend_name": { - "norms": false, - "type": "text" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "http_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "referrer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "ignore_above": 1024, - "type": "keyword" - }, - "request_count": { - "type": "long" - }, - "response_code": { - "type": "long" - }, - "url": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "index": false, - "norms": false, - "type": "text" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "url": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "full_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } -} diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index cb29695e306a8..22fe429dae04c 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -317,18 +317,7 @@ private void restoreMountAndVerify( Map valMapping = (Map) propertiesMapping.get("val"); assertThat(valMapping, hasKey("type")); assertEquals("long", valMapping.get("type")); - -// assertThat(root, hasKey("runtime")); -// assertThat(root.get("runtime"), instanceOf(Map.class)); -// @SuppressWarnings("unchecked") -// Map runtimeMapping = (Map) root.get("runtime"); -// assertThat(runtimeMapping, hasKey("val")); -// assertThat(runtimeMapping.get("val"), instanceOf(Map.class)); -// @SuppressWarnings("unchecked") -// Map valRuntimeMapping = (Map) runtimeMapping.get("val"); -// assertThat(valRuntimeMapping, hasKey("type")); -// assertEquals("long", valRuntimeMapping.get("type")); - + // run a search against the index assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository); From 22231a34a95a026eb88520415fddc17af8fa253a Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 25 Jan 2022 14:24:27 +0100 Subject: [PATCH 3/8] somefixes --- .../cluster/metadata/IndexMetadata.java | 8 +-- .../cluster/metadata/MappingMetadata.java | 10 ++++ .../snapshots/RestoreService.java | 32 +++------- .../oldrepos/OldRepositoryAccessIT.java | 60 ++++++++++++------- 4 files changed, 62 insertions(+), 48 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 71c3ec2b2f967..f37c4b5858aca 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -1935,7 +1935,7 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE } } else if (token == XContentParser.Token.START_ARRAY) { if ("mappings".equals(currentFieldName)) { - // don't try to parse these for now + MapBuilder mappingSourceBuilder = MapBuilder.newMapBuilder(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { Map mapping; if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { @@ -1944,8 +1944,9 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE } else { mapping = parser.mapOrdered(); } - handleLegacyMapping(builder, mapping); + mappingSourceBuilder.putAll(mapping); } + handleLegacyMapping(builder, mappingSourceBuilder.map()); } else { parser.skipChildren(); } @@ -1983,8 +1984,7 @@ private static void handleLegacyMapping(Builder builder, Map map String mappingType = mapping.keySet().iterator().next(); builder.putMapping(new MappingMetadata(mappingType, mapping)); } else if (mapping.size() > 1) { - // TODO: handle this better - throw new RuntimeException("can't handle multiple types"); + builder.putMapping(new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, mapping)); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java index cfe8a27074b60..ebd440dfc6c5c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java @@ -141,6 +141,16 @@ public Map getSourceAsMap() throws ElasticsearchParseException { return sourceAsMap(); } + /** + * Converts the serialized compressed form of the mappings into a parsed map. + * In contrast to {@link #sourceAsMap()}, this does not remove the type + */ + @SuppressWarnings("unchecked") + public Map rawSourceAsMap() throws ElasticsearchParseException { + Map mapping = XContentHelper.convertToMap(source.compressedReference(), true).v2(); + return mapping; + } + public boolean routingRequired() { return this.routingRequired; } diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index ff9e70aa533df..4bc162e2d9b6f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1296,8 +1296,8 @@ public ClusterState execute(ClusterState currentState) { request.indexSettings(), request.ignoreIndexSettings() ); - if (snapshotIndexMetadata.getCreationVersion().before( - currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion())) { + if (snapshotIndexMetadata.getCreationVersion() + .before(currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion())) { // adapt index metadata so that it can be understood by current version snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata); } @@ -1591,42 +1591,28 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) private IndexMetadata convertLegacyIndex(IndexMetadata snapshotIndexMetadata) { MappingMetadata mappingMetadata = snapshotIndexMetadata.mapping(); - Map loadedMappingSource = mappingMetadata.sourceAsMap(); + Map loadedMappingSource = mappingMetadata.rawSourceAsMap(); - // store old mapping under _meta/legacy-mapping + // store old mapping under _meta/legacy-mappings Map legacyMapping = new LinkedHashMap<>(); boolean sourceOnlySnapshot = snapshotIndexMetadata.getSettings().getAsBoolean("index.source_only", false); if (sourceOnlySnapshot) { - // actual mapping is under "_meta", and keyed by _type - Object sourceOnlyMeta = loadedMappingSource.get("_meta"); + // actual mapping is under "_meta" (but strip type first) + Object sourceOnlyMeta = mappingMetadata.sourceAsMap().get("_meta"); if (sourceOnlyMeta instanceof Map sourceOnlyMetaMap) { - if (sourceOnlyMetaMap.size() == 1) { - if (sourceOnlyMetaMap.values().iterator().next() instanceof Map sourceOnlyLegacyMapping) { - legacyMapping.put("legacy-mapping", sourceOnlyLegacyMapping); - } - } + legacyMapping.put("legacy-mappings", sourceOnlyMetaMap); } } else { - legacyMapping.put("legacy-mapping", loadedMappingSource); + legacyMapping.put("legacy-mappings", loadedMappingSource); } Map newMappingSource = new LinkedHashMap<>(); newMappingSource.put("_meta", legacyMapping); - // copy over _meta fields of original index into proper place? - // preserve legacy runtime fields - Object legacyRuntimeFields = loadedMappingSource.get("runtime"); - if (legacyRuntimeFields instanceof Map) { - @SuppressWarnings("unchecked") - Map legRuntimeFields = (Map) legacyRuntimeFields; - newMappingSource.put("runtime", legRuntimeFields); - } Map newMapping = new LinkedHashMap<>(); newMapping.put(mappingMetadata.type(), newMappingSource); // TODO: _routing? Perhaps we don't need to obey any routing here as stuff is read-only anyway and get API will be disabled - return IndexMetadata.builder(snapshotIndexMetadata) - .putMapping(new MappingMetadata(mappingMetadata.type(), newMapping)) - .build(); + return IndexMetadata.builder(snapshotIndexMetadata).putMapping(new MappingMetadata(mappingMetadata.type(), newMapping)).build(); } private static IndexMetadata.Builder restoreToCreateNewIndex(IndexMetadata snapshotIndexMetadata, String renamedIndexName) { diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index 22fe429dae04c..e519d90302730 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -61,10 +62,13 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; public class OldRepositoryAccessIT extends ESRestTestCase { @Override @@ -131,7 +135,9 @@ && randomBoolean()) { for (int i = 0; i < numDocs + extraDocs; i++) { String id = "testdoc" + i; expectedIds.add(id); - Request doc = new Request("PUT", "/test/doc/" + id); + // use multiple types for ES versions < 6.0.0 + String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + Request doc = new Request("PUT", "/test/" + type + "/" + id); doc.addParameter("refresh", "true"); doc.setJsonEntity(sourceForDoc(i)); assertOK(oldEs.performRequest(doc)); @@ -140,7 +146,8 @@ && randomBoolean()) { for (int i = 0; i < extraDocs; i++) { String id = randomFrom(expectedIds); expectedIds.remove(id); - Request doc = new Request("DELETE", "/test/doc/" + id); + String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + Request doc = new Request("DELETE", "/test/" + type + "/" + id); doc.addParameter("refresh", "true"); oldEs.performRequest(doc); } @@ -222,7 +229,7 @@ && randomBoolean()) { if (Build.CURRENT.isSnapshot()) { // restore / mount and check whether searches work - restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository); + restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository, oldVersion); // close indices assertTrue( @@ -240,7 +247,7 @@ && randomBoolean()) { ); // restore / mount again - restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository); + restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository, oldVersion); } } finally { IOUtils.closeWhileHandlingException( @@ -270,7 +277,8 @@ private void restoreMountAndVerify( Set expectedIds, RestHighLevelClient client, int numberOfShards, - boolean sourceOnlyRepository + boolean sourceOnlyRepository, + Version oldVersion ) throws IOException { // restore index RestoreSnapshotResponse restoreSnapshotResponse = client.snapshot() @@ -295,29 +303,39 @@ private void restoreMountAndVerify( .getStatus() ); - MappingMetadata mapping = client.indices().getMapping(new GetMappingsRequest().indices("restored_test"), RequestOptions.DEFAULT) - .mappings().get("restored_test"); + MappingMetadata mapping = client.indices() + .getMapping(new GetMappingsRequest().indices("restored_test"), RequestOptions.DEFAULT) + .mappings() + .get("restored_test"); logger.info("mapping for {}: {}", mapping.type(), mapping.source().string()); Map root = mapping.sourceAsMap(); assertThat(root, hasKey("_meta")); assertThat(root.get("_meta"), instanceOf(Map.class)); @SuppressWarnings("unchecked") Map meta = (Map) root.get("_meta"); - assertThat(meta, hasKey("legacy-mapping")); - assertThat(meta.get("legacy-mapping"), instanceOf(Map.class)); + assertThat(meta, hasKey("legacy-mappings")); + assertThat(meta.get("legacy-mappings"), instanceOf(Map.class)); @SuppressWarnings("unchecked") - Map legacyMapping = (Map) meta.get("legacy-mapping"); - assertThat(legacyMapping, hasKey("properties")); - assertThat(legacyMapping.get("properties"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map propertiesMapping = (Map) legacyMapping.get("properties"); - assertThat(propertiesMapping, hasKey("val")); - assertThat(propertiesMapping.get("val"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map valMapping = (Map) propertiesMapping.get("val"); - assertThat(valMapping, hasKey("type")); - assertEquals("long", valMapping.get("type")); - + Map legacyMappings = (Map) meta.get("legacy-mappings"); + assertThat(legacyMappings.keySet(), not(empty())); + for (Map.Entry entry : legacyMappings.entrySet()) { + String type = entry.getKey(); + assertThat(type, startsWith("doc")); + assertThat(entry.getValue(), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map legacyMapping = (Map) entry.getValue(); + assertThat(legacyMapping, hasKey("properties")); + assertThat(legacyMapping.get("properties"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map propertiesMapping = (Map) legacyMapping.get("properties"); + assertThat(propertiesMapping, hasKey("val")); + assertThat(propertiesMapping.get("val"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map valMapping = (Map) propertiesMapping.get("val"); + assertThat(valMapping, hasKey("type")); + assertEquals("long", valMapping.get("type")); + } + // run a search against the index assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository); From 1b29c638b47724b9bb002517097e2dc43aa7ca4b Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 26 Jan 2022 10:22:41 +0100 Subject: [PATCH 4/8] Access _type in archive data --- .../index/fieldvisitor/FieldsVisitor.java | 7 +- .../index/mapper/LegacyTypeFieldMapper.java | 90 +++++++++++++++++++ .../index/mapper/MapperRegistry.java | 9 +- .../oldrepos/OldRepositoryAccessIT.java | 44 +++++++-- 4 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 1556410ca0c2c..05192baf81cf9 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -68,7 +68,9 @@ public Status needsField(FieldInfo fieldInfo) { } // support _uid for loading older indices if ("_uid".equals(fieldInfo.name)) { - return Status.YES; + if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove("_type")) { + return Status.YES; + } } // All these fields are single-valued so we can stop when the set is // empty @@ -111,8 +113,9 @@ public void stringField(FieldInfo fieldInfo, String value) { if ("_uid".equals(fieldInfo.name)) { // 5.x-only int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can - // type = value.substring(0, delimiterIndex); + String type = value.substring(0, delimiterIndex); id = value.substring(delimiterIndex + 1); + addValue("_type", type); } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) { // only applies to 5.x indices that have single_type = true id = value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java new file mode 100644 index 0000000000000..9cb069ea030bd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.query.SearchExecutionContext; + +import java.util.Collections; + +public class LegacyTypeFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_type"; + + public static final String CONTENT_TYPE = "_type"; + + private static final LegacyTypeFieldMapper INSTANCE = new LegacyTypeFieldMapper(); + + public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); + + protected LegacyTypeFieldMapper() { + super(new LegacyTypeFieldType(), Lucene.KEYWORD_ANALYZER); + } + + public static class Defaults { + + public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType NESTED_FIELD_TYPE; + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setStored(true); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + + NESTED_FIELD_TYPE = new FieldType(); + NESTED_FIELD_TYPE.setTokenized(false); + NESTED_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + NESTED_FIELD_TYPE.setStored(false); + NESTED_FIELD_TYPE.setOmitNorms(true); + NESTED_FIELD_TYPE.freeze(); + } + } + + static final class LegacyTypeFieldType extends TermBasedFieldType { + + LegacyTypeFieldType() { + super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + protected boolean allowDocValueBasedQueries() { + return true; + } + + @Override + public boolean isSearchable() { + // The _type field is always searchable. + return true; + } + + @Override + public boolean mayExistInIndex(SearchExecutionContext context) { + return true; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new StoredValueFetcher(context.lookup(), NAME); + } + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java index 0ed5e7682a365..1173dd8f5489e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java @@ -26,6 +26,7 @@ public final class MapperRegistry { private final Map runtimeFieldParsers; private final Map metadataMapperParsers; private final Map metadataMapperParsers7x; + private final Map metadataMapperParsers5x; private final Function> fieldFilter; public MapperRegistry( @@ -40,6 +41,9 @@ public MapperRegistry( Map metadata7x = new LinkedHashMap<>(metadataMapperParsers); metadata7x.remove(NestedPathFieldMapper.NAME); this.metadataMapperParsers7x = metadata7x; + Map metadata5x = new LinkedHashMap<>(metadata7x); + metadata5x.put(LegacyTypeFieldMapper.NAME, LegacyTypeFieldMapper.PARSER); + this.metadataMapperParsers5x = metadata5x; this.fieldFilter = fieldFilter; } @@ -62,8 +66,11 @@ public Map getRuntimeFieldParsers() { public Map getMetadataMapperParsers(Version indexCreatedVersion) { if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { return metadataMapperParsers; + } else if (indexCreatedVersion.onOrAfter(Version.V_7_0_0)) { + return metadataMapperParsers7x; + } else { + return metadataMapperParsers5x; } - return metadataMapperParsers7x; } /** diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index e519d90302730..b7f471bd75916 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -136,7 +137,7 @@ && randomBoolean()) { String id = "testdoc" + i; expectedIds.add(id); // use multiple types for ES versions < 6.0.0 - String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + String type = getType(oldVersion, id); Request doc = new Request("PUT", "/test/" + type + "/" + id); doc.addParameter("refresh", "true"); doc.setJsonEntity(sourceForDoc(i)); @@ -146,7 +147,7 @@ && randomBoolean()) { for (int i = 0; i < extraDocs; i++) { String id = randomFrom(expectedIds); expectedIds.remove(id); - String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + String type = getType(oldVersion, id); Request doc = new Request("DELETE", "/test/" + type + "/" + id); doc.addParameter("refresh", "true"); oldEs.performRequest(doc); @@ -267,6 +268,10 @@ && randomBoolean()) { } } + private String getType(Version oldVersion, String id) { + return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + } + private static String sourceForDoc(int i) { return "{\"test\":\"test" + i + "\",\"val\":" + i + "}"; } @@ -337,7 +342,7 @@ private void restoreMountAndVerify( } // run a search against the index - assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository); + assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); // mount as full copy searchable snapshot RestoreSnapshotResponse mountSnapshotResponse = client.searchableSnapshots() @@ -363,7 +368,7 @@ private void restoreMountAndVerify( ); // run a search against the index - assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository); + assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); // mount as shared cache searchable snapshot mountSnapshotResponse = client.searchableSnapshots() @@ -378,11 +383,12 @@ private void restoreMountAndVerify( assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().successfulShards()); // run a search against the index - assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository); + assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); } @SuppressWarnings("removal") - private void assertDocs(String index, int numDocs, Set expectedIds, RestHighLevelClient client, boolean sourceOnlyRepository) + private void assertDocs(String index, int numDocs, Set expectedIds, RestHighLevelClient client, boolean sourceOnlyRepository, + Version oldVersion) throws IOException { // run a search against the index SearchResponse searchResponse = client.search(new SearchRequest(index), RequestOptions.DEFAULT); @@ -420,9 +426,9 @@ private void assertDocs(String index, int numDocs, Set expectedIds, Rest // check that doc values can be accessed by (reverse) sorting on numeric val field // first add mapping for field (this will be done automatically in the future) XContentBuilder mappingBuilder = JsonXContent.contentBuilder(); - mappingBuilder.startObject().startObject("properties").startObject("val"); - mappingBuilder.field("type", "long"); - mappingBuilder.endObject().endObject().endObject(); + mappingBuilder.startObject().startObject("properties"); + mappingBuilder.startObject("val").field("type", "long").endObject(); + mappingBuilder.endObject().endObject(); assertTrue( client.indices().putMapping(new PutMappingRequest(index).source(mappingBuilder), RequestOptions.DEFAULT).isAcknowledged() ); @@ -442,6 +448,26 @@ private void assertDocs(String index, int numDocs, Set expectedIds, Rest expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).collect(Collectors.toList()), Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()) ); + + // search on _type and check that results contain _type information + String randomType = getType(oldVersion, randomFrom(expectedIds)); + long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); + searchResponse = client.search( + new SearchRequest(index).source( + SearchSourceBuilder.searchSource() + .query(QueryBuilders.termQuery("_type", randomType)) + ) + , + RequestOptions.DEFAULT + ); + logger.info(searchResponse); + assertEquals(typeCount, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + DocumentField typeField = hit.field("_type"); + assertNotNull(typeField); + assertThat(typeField.getValue(), instanceOf(String.class)); + assertEquals(randomType, typeField.getValue()); + } } } From 9b9801ac5124477b0066e6dbd528362d73d4ff26 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 27 Jan 2022 10:00:40 +0100 Subject: [PATCH 5/8] fix --- .../index/mapper/LegacyTypeFieldMapper.java | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java index 9cb069ea030bd..1482a9ca62fee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -9,12 +9,20 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.sandbox.search.DocValuesTermsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.query.SearchExecutionContext; +import java.util.Collection; import java.util.Collections; +/** + * Field mapper to access the legacy _type that existed in Elasticsearch 5 + */ public class LegacyTypeFieldMapper extends MetadataFieldMapper { public static final String NAME = "_type"; @@ -62,14 +70,37 @@ public String typeName() { } @Override - protected boolean allowDocValueBasedQueries() { + public boolean isSearchable() { + // The _type field is always searchable. return true; } @Override - public boolean isSearchable() { - // The _type field is always searchable. - return true; + public Query termQuery(Object value, SearchExecutionContext context) { + return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value)); + } + + @Override + public Query termsQuery(Collection values, SearchExecutionContext context) { + BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new); + return new DocValuesTermsQuery(name(), bytesRefs); + } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + return SortedSetDocValuesField.newSlowRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper + ); } @Override From 460109f8245a61c1287c49dd51e2ca00d384a094 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 27 Jan 2022 10:17:53 +0100 Subject: [PATCH 6/8] more --- .../index/fieldvisitor/FieldsVisitor.java | 5 ++- .../index/mapper/MapperRegistry.java | 6 +-- .../oldrepos/OldRepositoryAccessIT.java | 45 ++++++++++--------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 05192baf81cf9..787d9ba7c7a6e 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.LegacyTypeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -68,7 +69,7 @@ public Status needsField(FieldInfo fieldInfo) { } // support _uid for loading older indices if ("_uid".equals(fieldInfo.name)) { - if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove("_type")) { + if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove(LegacyTypeFieldMapper.NAME)) { return Status.YES; } } @@ -115,7 +116,7 @@ public void stringField(FieldInfo fieldInfo, String value) { int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can String type = value.substring(0, delimiterIndex); id = value.substring(delimiterIndex + 1); - addValue("_type", type); + addValue(LegacyTypeFieldMapper.NAME, type); } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) { // only applies to 5.x indices that have single_type = true id = value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java index 1173dd8f5489e..034f056dda993 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java @@ -66,10 +66,10 @@ public Map getRuntimeFieldParsers() { public Map getMetadataMapperParsers(Version indexCreatedVersion) { if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { return metadataMapperParsers; - } else if (indexCreatedVersion.onOrAfter(Version.V_7_0_0)) { - return metadataMapperParsers7x; - } else { + } else if (indexCreatedVersion.major < 6) { return metadataMapperParsers5x; + } else { + return metadataMapperParsers7x; } } diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index 617cd68ba588a..10354a48c566d 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -387,9 +387,14 @@ private void restoreMountAndVerify( } @SuppressWarnings("removal") - private void assertDocs(String index, int numDocs, Set expectedIds, RestHighLevelClient client, boolean sourceOnlyRepository, - Version oldVersion) - throws IOException { + private void assertDocs( + String index, + int numDocs, + Set expectedIds, + RestHighLevelClient client, + boolean sourceOnlyRepository, + Version oldVersion + ) throws IOException { // run a search against the index SearchResponse searchResponse = client.search(new SearchRequest(index), RequestOptions.DEFAULT); logger.info(searchResponse); @@ -449,24 +454,22 @@ private void assertDocs(String index, int numDocs, Set expectedIds, Rest Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()) ); - // search on _type and check that results contain _type information - String randomType = getType(oldVersion, randomFrom(expectedIds)); - long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); - searchResponse = client.search( - new SearchRequest(index).source( - SearchSourceBuilder.searchSource() - .query(QueryBuilders.termQuery("_type", randomType)) - ) - , - RequestOptions.DEFAULT - ); - logger.info(searchResponse); - assertEquals(typeCount, searchResponse.getHits().getTotalHits().value); - for (SearchHit hit : searchResponse.getHits().getHits()) { - DocumentField typeField = hit.field("_type"); - assertNotNull(typeField); - assertThat(typeField.getValue(), instanceOf(String.class)); - assertEquals(randomType, typeField.getValue()); + if (oldVersion.before(Version.fromString("6.0.0"))) { + // search on _type and check that results contain _type information + String randomType = getType(oldVersion, randomFrom(expectedIds)); + long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); + searchResponse = client.search( + new SearchRequest(index).source(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType))), + RequestOptions.DEFAULT + ); + logger.info(searchResponse); + assertEquals(typeCount, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + DocumentField typeField = hit.field("_type"); + assertNotNull(typeField); + assertThat(typeField.getValue(), instanceOf(String.class)); + assertEquals(randomType, typeField.getValue()); + } } } } From 13cdfa5cc5c255c8ba0ad9197f7be09be9252bf9 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 27 Jan 2022 10:45:45 +0100 Subject: [PATCH 7/8] abs --- .../java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index 10354a48c566d..b8ab56bf69400 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -269,7 +269,7 @@ && randomBoolean()) { } private String getType(Version oldVersion, String id) { - return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0); } private static String sourceForDoc(int i) { From 0acde08c305f5fd51389dfb81d15b76db0593cc5 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 27 Jan 2022 12:29:49 +0100 Subject: [PATCH 8/8] remove dead code --- .../index/mapper/LegacyTypeFieldMapper.java | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java index 1482a9ca62fee..a5e0ec86db775 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -8,9 +8,7 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.sandbox.search.DocValuesTermsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -37,27 +35,6 @@ protected LegacyTypeFieldMapper() { super(new LegacyTypeFieldType(), Lucene.KEYWORD_ANALYZER); } - public static class Defaults { - - public static final FieldType FIELD_TYPE = new FieldType(); - public static final FieldType NESTED_FIELD_TYPE; - - static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setStored(true); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); - - NESTED_FIELD_TYPE = new FieldType(); - NESTED_FIELD_TYPE.setTokenized(false); - NESTED_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - NESTED_FIELD_TYPE.setStored(false); - NESTED_FIELD_TYPE.setOmitNorms(true); - NESTED_FIELD_TYPE.freeze(); - } - } - static final class LegacyTypeFieldType extends TermBasedFieldType { LegacyTypeFieldType() {