Check shard availability before including in stats (#96015) (#96016)

With this commit we check whether there is an available shard routing
before we test whether it has been started. This makes
`DataTiersUsageTransportAction` more resilient to potential temporary
inconsistencies between cluster state and node stats due to concurrent
shard movement.

Closes #87001
Closes #96000

Author: danielmitterdorfer

docs/changelog/96015.yaml

@@ -0,0 +1,7 @@
+pr: 96015
+summary: Check shard availability before including in stats
+area: Distributed 
+type: bug
+issues:
+ - 96000
+ - 87001

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java

@@ -206,7 +206,8 @@ private static void aggregateDataTierShardStats(NodeStats nodeStats, Index index
                 accumulator.docCount += shardStat.getTotal().getDocs().getCount();
 
                 // Accumulate stats about started shards
-                if (node.getByShardId(shardStat.getShardId()).state() == ShardRoutingState.STARTED) {
+                ShardRouting shardRouting = node.getByShardId(shardStat.getShardId());
+                if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) {
                     accumulator.totalShardCount += 1;
 
                     // Accumulate stats about started primary shards