Merge branch 'main' into changelogs-08-26

Author: lcawl

docs/changelog/157760.yaml

@@ -0,0 +1,5 @@
+area: Audit
+issues: []
+pr: 157760
+summary: "Audit: reject oversized request bodies"
+type: enhancement

docs/reference/elasticsearch/configuration-reference/auding-settings.md

@@ -68,7 +68,10 @@ $$$xpack-sa-lf-events-emit-request$$$
     Be advised that sensitive data may be audited in plain text when including the request body in audit events, even though all the security APIs, such as those that change the user’s password, have the credentials filtered out when audited.
     ::::
 
+$$$xpack-sa-lf-events-max-request-body-size$$$
 
+`xpack.security.audit.logfile.events.max_request_body_size` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
+:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) Maximum rendered JSON body size (in characters) that may be included in audit events when [`xpack.security.audit.logfile.events.emit_request_body`](#xpack-sa-lf-events-emit-request) is `true`. The limit is applied to the JSON representation of the request body (after format conversion), so it correctly accounts for formats that expand when converted to JSON. Requests whose rendered body exceeds this limit are rejected with HTTP 413 (Request Entity Too Large), ensuring the audit log is always a complete record of accepted requests. The default value is `2147483647b` (`Integer.MAX_VALUE`). Set to `0` to disable the limit entirely. Lower this value on nodes with limited available memory as needed.
 
 ## Local Node Info Settings [node-audit-settings]
 

modules/repository-s3/qa/deprecations/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3DeprecationsRestIT.java

@@ -18,6 +18,7 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
@@ -27,6 +28,9 @@
 import org.junit.rules.TestRule;
 
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 
@@ -140,6 +144,47 @@ public void testUpgradeAssistantReportsInsecureCredentials() throws IOException
         }
     }
 
+    private static final String PLACEHOLDER_CLIENT = "placeholder";
+    private static final Map<String, String> DEPRECATED_CLIENT_SETTING_TEST_VALUES = Map.of(
+        "protocol",
+        "https",
+        "use_throttle_retries",
+        "true",
+        "signer_override",
+        "test_signer"
+    );
+
+    public void testUpgradeAssistantReportsDeprecatedClientSettings() throws IOException {
+        for (var deprecatedClientSetting : DEPRECATED_CLIENT_SETTING_TEST_VALUES.entrySet()) {
+            final var settingKey = deprecatedClientSetting.getKey();
+            final var repoName = registerRepository(b -> b.put(settingKey, deprecatedClientSetting.getValue()), Strings.format("""
+                [s3.client.%s.%s] setting was deprecated in Elasticsearch and will be removed in a future release. \
+                See the breaking changes documentation for the next major version.""", PLACEHOLDER_CLIENT, settingKey));
+            try {
+                assertDeprecationIssue(
+                    repoName,
+                    "S3 repository explicitly configures a deprecated client setting",
+                    ReferenceDocs.TROUBLESHOOT_REPOSITORY,
+                    S3Repository.deprecatedClientSettingDeprecationWarning(settingKey)
+                );
+            } finally {
+                assertOK(client().performRequest(new Request("DELETE", "/_snapshot/" + repoName)));
+            }
+        }
+    }
+
+    public void testAllDeprecatedClientSettingsAreCoveredByUpgradeAssistantTest() {
+        final Set<String> deprecatedClientSettings = new HashSet<>();
+        for (final var setting : S3RepositorySettings.DEPRECATED_CLIENT_SETTINGS) {
+            deprecatedClientSettings.add(
+                setting.getConcreteSettingForNamespace(PLACEHOLDER_CLIENT)
+                    .getKey()
+                    .substring(S3ClientSettings.REPOSITORY_CLIENT_SETTINGS_PREFIX.length())
+            );
+        }
+        assertThat(DEPRECATED_CLIENT_SETTING_TEST_VALUES.keySet(), equalTo(deprecatedClientSettings));
+    }
+
     private static void assertDeprecationIssue(String repositoryName, String message, ReferenceDocs referenceDocs, String details)
         throws IOException {
         final var deprecations = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_migration/deprecations")));

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java

@@ -52,6 +52,11 @@ final class S3ClientSettings {
 
     /** Placeholder client name for normalizing client settings in the repository settings. */
     private static final String PLACEHOLDER_CLIENT = "placeholder";
+    static final String REPOSITORY_CLIENT_SETTINGS_PREFIX = PREFIX + PLACEHOLDER_CLIENT + '.';
+
+    static Settings normalizeRepositorySettings(Settings repositorySettings) {
+        return Settings.builder().put(repositorySettings).normalizePrefix(REPOSITORY_CLIENT_SETTINGS_PREFIX).build();
+    }
 
     /** The access key (ie login id) for connecting to s3. */
     static final Setting.AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(
@@ -349,10 +354,7 @@ private S3ClientSettings(
      */
     S3ClientSettings refine(Settings repositorySettings) {
         // Normalize settings to placeholder client settings prefix so that we can use the affix settings directly
-        final Settings normalizedSettings = Settings.builder()
-            .put(repositorySettings)
-            .normalizePrefix(PREFIX + PLACEHOLDER_CLIENT + '.')
-            .build();
+        final Settings normalizedSettings = normalizeRepositorySettings(repositorySettings);
         final HttpScheme newProtocol = getRepoSettingOrDefault(PROTOCOL_SETTING, normalizedSettings, protocol);
         final String newEndpoint = getRepoSettingOrDefault(ENDPOINT_SETTING, normalizedSettings, endpoint);
 
@@ -399,6 +401,11 @@ S3ClientSettings refine(Settings repositorySettings) {
             tenaciousRetriesEnabled
         );
         final boolean newAlwaysSignRequests = getRepoSettingOrDefault(ALWAYS_SIGN_REQUESTS, normalizedSettings, alwaysSignRequests);
+
+        // Read unused settings too so repository registration emits deprecation warnings for explicit values.
+        getRepoSettingOrDefault(UNUSED_USE_THROTTLE_RETRIES_SETTING, normalizedSettings, true);
+        getRepoSettingOrDefault(UNUSED_SIGNER_OVERRIDE, normalizedSettings, "");
+
         if (Objects.equals(protocol, newProtocol)
             && Objects.equals(endpoint, newEndpoint)
             && Objects.equals(proxyHost, newProxyHost)

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

@@ -408,6 +408,13 @@ class S3Repository extends MeteredBlobStoreRepository {
         UNSAFELY_INCOMPATIBLE_WITH_S3_CONDITIONAL_WRITES.getKey()
     );
 
+    static String deprecatedClientSettingDeprecationWarning(String settingKey) {
+        return Strings.format(
+            "This repository's settings include the deprecated S3 client setting [%s] which must be removed before upgrade.",
+            settingKey
+        );
+    }
+
     @Override
     public Collection<RepositoryDeprecationInfo> getDeprecationInfos() {
         final List<RepositoryDeprecationInfo> deprecationInfos = new ArrayList<>();
@@ -423,6 +430,24 @@ public Collection<RepositoryDeprecationInfo> getDeprecationInfos() {
                 )
             );
         }
+        for (String normalizedKey : S3ClientSettings.normalizeRepositorySettings(getMetadata().settings()).keySet()) {
+            for (Setting.AffixSetting<?> setting : S3RepositorySettings.DEPRECATED_CLIENT_SETTINGS) {
+                if (setting.match(normalizedKey)) {
+                    deprecationInfos.add(
+                        new RepositoryDeprecationInfo(
+                            RepositoryDeprecationInfo.Level.CRITICAL,
+                            "S3 repository explicitly configures a deprecated client setting",
+                            ReferenceDocs.TROUBLESHOOT_REPOSITORY,
+                            deprecatedClientSettingDeprecationWarning(
+                                normalizedKey.substring(S3ClientSettings.REPOSITORY_CLIENT_SETTINGS_PREFIX.length())
+                            ),
+                            false
+                        )
+                    );
+                    break;
+                }
+            }
+        }
         if (UNSAFELY_INCOMPATIBLE_WITH_S3_CONDITIONAL_WRITES.exists(getMetadata().settings())) {
             deprecationInfos.add(
                 new RepositoryDeprecationInfo(

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java

@@ -34,7 +34,6 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -132,37 +131,7 @@ public Map<String, Repository.Factory> getRepositories(
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Arrays.asList(
-            // named s3 client configuration settings
-            S3ClientSettings.ACCESS_KEY_SETTING,
-            S3ClientSettings.SECRET_KEY_SETTING,
-            S3ClientSettings.SESSION_TOKEN_SETTING,
-            S3ClientSettings.ENDPOINT_SETTING,
-            S3ClientSettings.PROTOCOL_SETTING,
-            S3ClientSettings.PROXY_HOST_SETTING,
-            S3ClientSettings.PROXY_PORT_SETTING,
-            S3ClientSettings.PROXY_SCHEME_SETTING,
-            S3ClientSettings.PROXY_USERNAME_SETTING,
-            S3ClientSettings.PROXY_PASSWORD_SETTING,
-            S3ClientSettings.READ_TIMEOUT_SETTING,
-            S3ClientSettings.MAX_CONNECTIONS_SETTING,
-            S3ClientSettings.MAX_RETRIES_SETTING,
-            S3ClientSettings.API_CALL_TIMEOUT_SETTING,
-            S3ClientSettings.UNUSED_USE_THROTTLE_RETRIES_SETTING,
-            S3ClientSettings.USE_PATH_STYLE_ACCESS,
-            S3ClientSettings.DISABLE_CHUNKED_ENCODING,
-            S3ClientSettings.UNUSED_SIGNER_OVERRIDE,
-            S3ClientSettings.ADD_PURPOSE_CUSTOM_QUERY_PARAMETER,
-            S3ClientSettings.REGION,
-            S3ClientSettings.CONNECTION_MAX_IDLE_TIME_SETTING,
-            S3ClientSettings.MAX_COPY_SIZE_BEFORE_MULTIPART,
-            S3Service.REPOSITORY_S3_CAS_TTL_SETTING,
-            S3Service.REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING,
-            S3Repository.ACCESS_KEY_SETTING,
-            S3Repository.SECRET_KEY_SETTING,
-            S3ClientSettings.S3_TENACIOUS_RETRIES_ENABLED_SETTING,
-            S3ClientSettings.ALWAYS_SIGN_REQUESTS
-        );
+        return S3RepositorySettings.SETTINGS;
     }
 
     @Override

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositorySettings.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import org.elasticsearch.common.settings.Setting;
+
+import java.util.ArrayList;
+import java.util.List;
+
+final class S3RepositorySettings {
+
+    static final List<Setting<?>> SETTINGS = List.of(
+        // named s3 client configuration settings
+        S3ClientSettings.ACCESS_KEY_SETTING,
+        S3ClientSettings.SECRET_KEY_SETTING,
+        S3ClientSettings.SESSION_TOKEN_SETTING,
+        S3ClientSettings.ENDPOINT_SETTING,
+        S3ClientSettings.PROTOCOL_SETTING,
+        S3ClientSettings.PROXY_HOST_SETTING,
+        S3ClientSettings.PROXY_PORT_SETTING,
+        S3ClientSettings.PROXY_SCHEME_SETTING,
+        S3ClientSettings.PROXY_USERNAME_SETTING,
+        S3ClientSettings.PROXY_PASSWORD_SETTING,
+        S3ClientSettings.READ_TIMEOUT_SETTING,
+        S3ClientSettings.MAX_CONNECTIONS_SETTING,
+        S3ClientSettings.MAX_RETRIES_SETTING,
+        S3ClientSettings.API_CALL_TIMEOUT_SETTING,
+        S3ClientSettings.UNUSED_USE_THROTTLE_RETRIES_SETTING,
+        S3ClientSettings.USE_PATH_STYLE_ACCESS,
+        S3ClientSettings.DISABLE_CHUNKED_ENCODING,
+        S3ClientSettings.UNUSED_SIGNER_OVERRIDE,
+        S3ClientSettings.ADD_PURPOSE_CUSTOM_QUERY_PARAMETER,
+        S3ClientSettings.REGION,
+        S3ClientSettings.CONNECTION_MAX_IDLE_TIME_SETTING,
+        S3ClientSettings.MAX_COPY_SIZE_BEFORE_MULTIPART,
+        S3Service.REPOSITORY_S3_CAS_TTL_SETTING,
+        S3Service.REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING,
+        S3Repository.ACCESS_KEY_SETTING,
+        S3Repository.SECRET_KEY_SETTING,
+        S3ClientSettings.S3_TENACIOUS_RETRIES_ENABLED_SETTING,
+        S3ClientSettings.ALWAYS_SIGN_REQUESTS
+    );
+
+    static final List<Setting.AffixSetting<?>> DEPRECATED_CLIENT_SETTINGS = deprecatedClientSettings();
+
+    private S3RepositorySettings() {}
+
+    private static List<Setting.AffixSetting<?>> deprecatedClientSettings() {
+        final List<Setting.AffixSetting<?>> deprecatedClientSettings = new ArrayList<>();
+        for (Setting<?> setting : SETTINGS) {
+            if (setting instanceof Setting.AffixSetting<?> affixSetting && setting.getProperties().contains(Setting.Property.Deprecated)) {
+                deprecatedClientSettings.add(affixSetting);
+            }
+        }
+        return List.copyOf(deprecatedClientSettings);
+    }
+}

server/src/main/java/org/elasticsearch/escf/LuceneLongColumn.java

@@ -14,6 +14,7 @@
 import org.apache.lucene.document.column.LongColumn;
 import org.apache.lucene.document.column.LongTupleCursor;
 import org.apache.lucene.document.column.LongValuesCursor;
+import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.util.BytesRef;
@@ -187,6 +188,23 @@ void setDocValue(long v) {
             fieldsData = v;
         }
 
+        @Override
+        public Number numericValue() {
+            final long raw = (Long) fieldsData;
+            // For stored-only fields (no doc values), the indexing chain calls numericValue().floatValue()
+            // or .doubleValue() to retrieve the actual value. Return the correctly-typed Number so that
+            // the stored representation matches what StoredField would produce.
+            if (fieldType().stored() && fieldType().docValuesType() == DocValuesType.NONE) {
+                return switch (kind) {
+                    case LONG -> raw;
+                    case INT -> (int) raw;
+                    case FLOAT -> NumericUtils.sortableIntToFloat((int) raw);
+                    case DOUBLE -> NumericUtils.sortableLongToDouble(raw);
+                };
+            }
+            return raw;
+        }
+
         @Override
         public BytesRef binaryValue() {
             // Consulted by the indexing chain only when fieldType.pointDimensionCount() > 0.

server/src/main/java/org/elasticsearch/escf/NumberColumnTransform.java

@@ -53,6 +53,25 @@ public static EscfColumnData toHalfFloatPointBinaryColumn(EscfColumn source, Rec
         return builder.finish(source.docCount());
     }
 
+    /**
+     * Converts a LONG {@link EscfColumn} whose values are
+     * {@link HalfFloatPoint#halfFloatToSortableShort} encoded sortable shorts into a LONG
+     * {@link EscfColumnData} containing {@link NumericUtils#floatToSortableInt} encoded sortable ints
+     * (widened to long). Use the result with a {@link org.elasticsearch.escf.LuceneLongColumn} and
+     * {@link org.apache.lucene.document.column.LongColumn.NumericKind#FLOAT} to emit the stored-fields
+     * column for a {@code half_float} field.
+     */
+    public static EscfColumnData toHalfFloatStoredLongColumn(EscfColumn source, Recycler<BytesRef> recycler) {
+        assert source.kind() == EscfColumnKind.LONG : "expected LONG, got " + EscfColumnKind.name(source.kind());
+        EscfColumnBuilder builder = newLongBuilder(recycler);
+        LongTupleCursor cursor = source.longCursor();
+        for (int doc = cursor.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = cursor.nextDoc()) {
+            float f = HalfFloatPoint.sortableShortToHalfFloat((short) cursor.longValue());
+            builder.setLong(doc, NumericUtils.floatToSortableInt(f));
+        }
+        return builder.finish(source.docCount());
+    }
+
     public static EscfColumnData toSortableLongColumn(
         EscfColumn source,
         NumberFieldMapper.NumberType type,