Open
Description
Engine.getSparseVectorValueCount might block 100% of the management threads in case we're facing frequent cache misses.
This causes the thread pool to queue and potentially starve important other work that's also depending on the management thread pool.
This was discovered on an internal serverless project.
Here's the corresponding hot_threads output:
0.3% [cpu=0.3%, other=0.0%] (1.3ms out of 500ms) cpu usage by thread 'elasticsearch[es-es-index-546895bb95-9l8qb][management][T#2]'
10/10 snapshots sharing following 98 elements
java.base@24/jdk.internal.misc.Unsafe.park(Native Method)
java.base@24/java.util.concurrent.locks.LockSupport.park(LockSupport.java:223)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:789)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1138)
app/[email protected]/org.elasticsearch.action.support.PlainActionFuture$Sync.get(PlainActionFuture.java:265)
app/[email protected]/org.elasticsearch.action.support.PlainActionFuture.get(PlainActionFuture.java:96)
[email protected]/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.readSingleRegion(SharedBlobCacheService.java:1252)
[email protected]/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.populateAndRead(SharedBlobCacheService.java:1229)
[email protected]/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.doRead(CacheFileReader.java:124)
[email protected]/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.read(CacheFileReader.java:94)
[email protected]/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternalSlow(BlobCacheIndexInput.java:175)
[email protected]/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.doReadInternal(BlobCacheIndexInput.java:161)
[email protected]/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternal(BlobCacheIndexInput.java:131)
[email protected]/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.refill(BlobCacheBufferedIndexInput.java:304)
[email protected]/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readByte(BlobCacheBufferedIndexInput.java:57)
app/[email protected]/org.apache.lucene.store.DataInput.readVInt(DataInput.java:121)
[email protected]/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readVInt(BlobCacheBufferedIndexInput.java:172)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:174)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.lambda$prepareSeekExact$1(SegmentTermsEnum.java:511)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum$$Lambda/0x0000000030b2a4a8.get(Unknown Source)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:599)
app/[email protected]/org.elasticsearch.index.engine.Engine.getSparseVectorValueCount(Engine.java:395)
app/[email protected]/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:379)
app/[email protected]/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:369)
app/[email protected]/org.elasticsearch.index.shard.IndexShard.sparseVectorStats(IndexShard.java:1482)
app/[email protected]/org.elasticsearch.action.admin.indices.stats.CommonStats.getShardLevelStats(CommonStats.java:192)
app/[email protected]/org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:564)
app/[email protected]/org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:518)
app/[email protected]/org.elasticsearch.indices.IndicesService.stats(IndicesService.java:489)
app/[email protected]/org.elasticsearch.node.NodeService.stats(NodeService.java:188)
app/[email protected]/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:161)
app/[email protected]/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:47)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperationAsync(TransportNodesAction.java:245)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.lambda$messageReceived$0(TransportNodesAction.java:261)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler$$Lambda/0x00000000308c7978.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListener.run(ActionListener.java:465)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:579)
app/[email protected]/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:632)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:621)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:470)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030802fa8.accept(Unknown Source)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1062)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1028)
app/[email protected]/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$9(AuthorizationService.java:484)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030803438.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:211)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:474)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:450)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$3(AuthorizationService.java:337)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x00000000307f4fe8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:258)
app/[email protected]/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$0(RBACEngine.java:176)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda/0x00000000307f5478.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRoles$4(CompositeRolesStore.java:213)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x00000000307f56a8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$6(CompositeRolesStore.java:232)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b768.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$0(RoleReferenceIntersection.java:49)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087a7e8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/[email protected]/org.elasticsearch.action.support.GroupedActionListener.onResponse(GroupedActionListener.java:57)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.buildRoleFromRoleReference(CompositeRolesStore.java:330)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$5(CompositeRolesStore.java:231)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b520.accept(Unknown Source)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$1(RoleReferenceIntersection.java:53)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087aa18.accept(Unknown Source)
java.base@24/java.lang.Iterable.forEach(Iterable.java:75)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.buildRole(RoleReferenceIntersection.java:53)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRole(CompositeRolesStore.java:230)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:206)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:172)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:353)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter.lambda$inbound$1(ServerTransportFilter.java:114)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter$$Lambda/0x0000000030639ab8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$MappedActionListener.onResponse(ActionListenerImplementations.java:111)
[email protected]/org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticate(AuthenticatorChain.java:93)
[email protected]/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:271)
[email protected]/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:203)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter.authenticate(ServerTransportFilter.java:127)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter.inbound(ServerTransportFilter.java:105)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:643)
app/[email protected]/org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:90)
app/[email protected]/org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:1092)
app/[email protected]/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:1044)
app/[email protected]/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
0.2% [cpu=0.2%, other=0.0%] (933.6micros out of 500ms) cpu usage by thread 'elasticsearch[es-es-index-546895bb95-9l8qb][management][T#1]'
10/10 snapshots sharing following 98 elements
java.base@24/jdk.internal.misc.Unsafe.park(Native Method)
java.base@24/java.util.concurrent.locks.LockSupport.park(LockSupport.java:223)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:789)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1138)
app/[email protected]/org.elasticsearch.action.support.PlainActionFuture$Sync.get(PlainActionFuture.java:265)
app/[email protected]/org.elasticsearch.action.support.PlainActionFuture.get(PlainActionFuture.java:96)
[email protected]/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.readSingleRegion(SharedBlobCacheService.java:1252)
[email protected]/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.populateAndRead(SharedBlobCacheService.java:1229)
[email protected]/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.doRead(CacheFileReader.java:124)
[email protected]/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.read(CacheFileReader.java:94)
[email protected]/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternalSlow(BlobCacheIndexInput.java:175)
[email protected]/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.doReadInternal(BlobCacheIndexInput.java:161)
[email protected]/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternal(BlobCacheIndexInput.java:131)
[email protected]/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.refill(BlobCacheBufferedIndexInput.java:304)
[email protected]/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readByte(BlobCacheBufferedIndexInput.java:57)
app/[email protected]/org.apache.lucene.store.DataInput.readVInt(DataInput.java:121)
[email protected]/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readVInt(BlobCacheBufferedIndexInput.java:172)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:174)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.lambda$prepareSeekExact$1(SegmentTermsEnum.java:511)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum$$Lambda/0x0000000030b2a4a8.get(Unknown Source)
app/[email protected]/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:599)
app/[email protected]/org.elasticsearch.index.engine.Engine.getSparseVectorValueCount(Engine.java:395)
app/[email protected]/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:379)
app/[email protected]/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:369)
app/[email protected]/org.elasticsearch.index.shard.IndexShard.sparseVectorStats(IndexShard.java:1482)
app/[email protected]/org.elasticsearch.action.admin.indices.stats.CommonStats.getShardLevelStats(CommonStats.java:192)
app/[email protected]/org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:564)
app/[email protected]/org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:518)
app/[email protected]/org.elasticsearch.indices.IndicesService.stats(IndicesService.java:489)
app/[email protected]/org.elasticsearch.node.NodeService.stats(NodeService.java:188)
app/[email protected]/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:161)
app/[email protected]/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:47)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperationAsync(TransportNodesAction.java:245)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.lambda$messageReceived$0(TransportNodesAction.java:261)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler$$Lambda/0x00000000308c7978.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListener.run(ActionListener.java:465)
app/[email protected]/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:579)
app/[email protected]/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:632)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:621)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:470)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030802fa8.accept(Unknown Source)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1062)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1028)
app/[email protected]/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$9(AuthorizationService.java:484)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030803438.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:211)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:474)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:450)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$3(AuthorizationService.java:337)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x00000000307f4fe8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:258)
app/[email protected]/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$0(RBACEngine.java:176)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda/0x00000000307f5478.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRoles$4(CompositeRolesStore.java:213)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x00000000307f56a8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$6(CompositeRolesStore.java:232)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b768.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$0(RoleReferenceIntersection.java:49)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087a7e8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/[email protected]/org.elasticsearch.action.support.GroupedActionListener.onResponse(GroupedActionListener.java:57)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.buildRoleFromRoleReference(CompositeRolesStore.java:330)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$5(CompositeRolesStore.java:231)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b520.accept(Unknown Source)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$1(RoleReferenceIntersection.java:53)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087aa18.accept(Unknown Source)
java.base@24/java.lang.Iterable.forEach(Iterable.java:75)
[email protected]/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.buildRole(RoleReferenceIntersection.java:53)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRole(CompositeRolesStore.java:230)
[email protected]/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:206)
[email protected]/org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:172)
[email protected]/org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:353)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter.lambda$inbound$1(ServerTransportFilter.java:114)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter$$Lambda/0x0000000030639ab8.accept(Unknown Source)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/[email protected]/org.elasticsearch.action.ActionListenerImplementations$MappedActionListener.onResponse(ActionListenerImplementations.java:111)
[email protected]/org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticate(AuthenticatorChain.java:93)
[email protected]/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:271)
[email protected]/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:203)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter.authenticate(ServerTransportFilter.java:127)
[email protected]/org.elasticsearch.xpack.security.transport.ServerTransportFilter.inbound(ServerTransportFilter.java:105)
[email protected]/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:643)
app/[email protected]/org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:90)
app/[email protected]/org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:1092)
app/[email protected]/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:1044)
app/[email protected]/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)