Merge branch 'main' into email_domains

.backportrc.json

@@ -2,6 +2,8 @@
   "upstream": "elastic/ecs",
   "branches": [
     { "name": "main", "checked": true },
+    "8.x",
+    "8.16",
     "8.11",
     "8.10",
     "8.9",

.github/workflows/test.yml

@@ -7,8 +7,8 @@ jobs:
     runs-on: ubuntu-22.04
     name: Unit Tests
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
         with:
           python-version: '3.x'
       - run: git fetch --prune --unshallow --tags

CHANGELOG.md

@@ -3,6 +3,35 @@
 # CHANGELOG
 All notable changes to this project will be documented in this file based on the [Keep a Changelog](http://keepachangelog.com/) Standard. This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [8.16.0](https://github.com/elastic/ecs/compare/v8.11.0...v8.16.0)
+
+### Schema Changes
+
+#### Bugfixes
+* Fix broken link in docs for vulnerability.id. #2328
+
+#### Added
+
+* Added `volume.*` as beta field set. #2269
+* Advanced `process.env_vars` to GA. #2315
+* Advanced `process.io` and `process.tty` fields to GA. #2317
+* Added `threat.indicator.id`. #2324
+* Added `process.group` to generated schemas. #2335
+
+#### Improvements
+
+* Define base encoding of `x509.serial_number`. #2383
+
+### Tooling and Artifact Changes
+
+#### Bugfixes
+
+* Fix broken link for vulnerabilty.id #2328
+
+#### Added
+
+* Documentation in README.md providing instruction on contributions to ECS during the OTel donation #2325
+
 ## [8.11.0](https://github.com/elastic/ecs/compare/v8.10.0...v8.11.0)
 
 ### Schema Changes

CHANGELOG.next.md

@@ -13,7 +13,6 @@ Thanks, you're awesome :-) -->
 #### Breaking changes
 
 #### Bugfixes
-* Fix broken link in docs for vulnerability.id. #2328
 
 #### Added
 
@@ -27,6 +26,7 @@ Thanks, you're awesome :-) -->
 #### Improvements
 
 * Define base encoding of `x509.serial_number`. #2383
+* Restrict the encoding of `x509.serial_number` to base 16. #2398
 
 #### Deprecated
 
@@ -38,8 +38,6 @@ Thanks, you're awesome :-) -->
 
 #### Added
 
-* Documentation in README.md providing instruction on contributions to ECS during the OTel donation #2325
-
 #### Improvements
 
 #### Deprecated

Makefile

@@ -8,6 +8,7 @@ PYTHON           := build/ve/bin/python
 SUBSETS_DIR      := schemas/subsets/
 EXP_SUBSETS_DIR  := experimental/schemas/subsets/
 VERSION          := $(shell cat version)
+SEMCONV_VERSION  := $(shell cat otel-semconv-version)
 
 #
 # Targets (sorted alphabetically)
@@ -51,7 +52,7 @@ docs:
 # Alias to generate experimental artifacts
 .PHONY: experimental
 experimental: ve
-	$(PYTHON) scripts/generator.py --include experimental/schemas --subset "${SUBSETS_DIR}" "${EXP_SUBSETS_DIR}" --out experimental
+	$(PYTHON) scripts/generator.py --include experimental/schemas --subset "${SUBSETS_DIR}" "${EXP_SUBSETS_DIR}" --semconv-version "${SEMCONV_VERSION}" --out experimental
 
 # Format code and files in the repo.
 .PHONY: fmt
@@ -66,7 +67,7 @@ generate: generator
 # Run the new generator
 .PHONY: generator
 generator: ve
-	$(PYTHON) scripts/generator.py --strict --include "${INCLUDE}" --subset "${SUBSETS_DIR}" --force-docs
+	$(PYTHON) scripts/generator.py --strict --include "${INCLUDE}" --subset "${SUBSETS_DIR}" --semconv-version "${SEMCONV_VERSION}" --force-docs
 
 # Check Makefile format.
 .PHONY: makelint

USAGE.md

@@ -372,7 +372,9 @@ This is an example `template.json` to be passed with `--template-setting-legacy`
       "refresh_interval": "1s"
     }
   },
-  "mappings": {}
+  "template": {
+    "mappings": {}
+  }
 }
 ```
 

docs/fields/fields.asciidoc

@@ -1,7 +1,7 @@
 [[ecs-field-reference]]
 == {ecs} Field Reference
 
-This is the documentation of ECS version 8.12.0-dev.
+This is the documentation of ECS version 9.0.0-dev.
 
 ECS defines multiple groups of related fields. They are called "field sets".
 The <<ecs-base,Base>> field set is the only one whose fields are defined

docs/index.asciidoc

@@ -13,7 +13,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
 [[ecs-reference]]
 == Overview
 
-This is the documentation of ECS version 8.12.0-dev.
+This is the documentation of ECS version 9.0.0-dev.
 
 [float]
 === What is ECS?
@@ -69,5 +69,6 @@ include::using-ecs/index.asciidoc[]
 include::fields/fields.asciidoc[]
 include::fields/field-values.asciidoc[]
 include::migrating/index.asciidoc[]
+include::opentelemetry/index.asciidoc[]
 include::additional-info/index.asciidoc[]
 include::release-notes/index.asciidoc[]

docs/opentelemetry/index.asciidoc

@@ -0,0 +1,69 @@
+[[ecs-opentelemetry]]
+== ECS & OpenTelemetry
+
+In April 2023, Elastic donated ECS to OpenTelemetry and together with the OTel community https://opentelemetry.io/blog/2023/ecs-otel-semconv-convergence/[jointly announced]
+the intention to achieve convergence of ECS and https://opentelemetry.io/docs/specs/semconv/[OTel Semantic Conventions (SemConv)].
+This donation should be seen as a directional decision for the evolution of both standards
+rather than a single event that merged both schemas into a single standard.
+
+While both schemes complement each other in large areas and thus offer mutual added value and great potential for convergence,
+it is important to understand that in some areas convergence is not achievable due to conceptual differences or consistency reasons. 
+For example, while ECS contains a few fields (e.g. https://www.elastic.co/guide/en/ecs/current/ecs-container.html#field-container-disk-read-bytes[container.disk.read.bytes])
+that are meant to be used in a metric-like way, metrics in OpenTelemetry are following a completely different data model (with metric name, type, dimensions, etc.).
+Also, in OTel semantic conventions there are some stable (or quasi-stable) attributes that have semantically equivalent, stable fields in ECS, however, with different field names.
+This kind of differences require explicit handling to achieve compatibility.
+
+[float]
+[[ecs-opentelemetry-relation]]
+=== Relation between ECS and Semantic Conventions
+
+The https://github.com/elastic/ecs/tree/main/schemas[ECS schema files] contain an explicit mapping between ECS fields and corresponding OTel semnatic convention attributes.
+This can be used to generate tooling for compatibility between ECS and semnatic conventions (e.g. alias fields in Elasticsearch).
+The relation between individual ECS fields and corresponding OTel semantic conventions attributes follows one of the following categories:
+
+[%header]
+|====
+| Category | Description
+
+| image:https://img.shields.io/badge/match-93c93e?style=flat[relation,title=match]
+| The name of the ECS field is identical to the SemConv attribute name and has (practically) the same semantics. 
+
+| image:https://img.shields.io/badge/equivalent-1ba9f5?style=flat[relation,title=equivalent]
+| The ECS field name is different but has the same semantics as the corresponding SemConv attribute.
+For this type of relation aliasing approaches (e.g. https://www.elastic.co/guide/en/elasticsearch/reference/current/field-alias.html[Elasticsearch field aliases]) can be used to achieve compatibility between ECS and OTel SemConv.
+
+| image:https://img.shields.io/badge/related-efc20d?style=flat[relation,title=related]
+| The ECS field name is different and has related - yet different - semantics as the corresponding SemConv attribute.
+An aliasing approach is not sufficient to resolve compatibility for this type of relation. 
+
+| image:https://img.shields.io/badge/conflict-910000?style=flat[relation,title=conflict]
+| The ECS field name is the same as an OTel SemConv namespace or an attribute that has significantly different semantics.
+
+| image:https://img.shields.io/badge/OTLP-ffdcb2?style=flat[relation,title=OTLP]
+| The ECS field has a corresponding representation in https://github.com/open-telemetry/opentelemetry-proto[OpenTelemetry's protocol definition].
+
+| image:https://img.shields.io/badge/metric-cb00cb?style=flat[relation,title=metric]
+| For this ECS field there is a corresponding metric defined in OTel SemConv.
+
+| image:https://img.shields.io/badge/n%2Fa-f2f4fb?style=flat[relation,title=na]
+| The ECS field is not applicable in the context of OTel or won't be aligned due to significant, conceptual conflict with OTel concepts in that area.
+|====
+
+
+
+The following documentation pages provide an overview and more details on the alignment between ECS and OTel semantic conventions:
+
+- <<ecs-otel-alignment-overview>>
+- <<ecs-otel-alignment-details>>
+
+
+
+[[ecs-otel-alignment-overview]]
+=== OTel Alignment Overview
+
+include::otel-mapping-summary.asciidoc[]
+
+[[ecs-otel-alignment-details]]
+=== Field & Attributes Alignment
+
+include::otel-fields-mapping.asciidoc[]