save

Author: xy720

be/src/agent/task_worker_pool.cpp

@@ -111,6 +111,7 @@ std::mutex s_task_signatures_mtx;
 std::unordered_map<TTaskType::type, std::unordered_set<int64_t>> s_task_signatures;
 
 std::atomic_ulong s_report_version(time(nullptr) * 100000);
+std::atomic<int64_t> s_tablet_report_failure_start_time(0);
 
 void increase_report_version() {
     s_report_version.fetch_add(1, std::memory_order_relaxed);
@@ -1181,13 +1182,20 @@ void report_tablet_callback(StorageEngine& engine, const ClusterInfo* cluster_in
         }
     }
 
-    if (report_version < s_report_version) {
+    if (report_version < s_report_version || UNLIKELY(config::enable_debug_points &&
+                 DebugPoints::instance()->is_enable("WorkPoolReportTablet.report_tablet_callback.skip"))) {
         // TODO llj This can only reduce the possibility for report error, but can't avoid it.
         // If FE create a tablet in FE meta and send CREATE task to this BE, the tablet may not be included in this
         // report, and the report version has a small probability that it has not been updated in time. When FE
         // receives this report, it is possible to delete the new tablet.
         LOG(WARNING) << "report version " << report_version << " change to " << s_report_version;
         DorisMetrics::instance()->report_all_tablets_requests_skip->increment(1);
+        int64_t current_time = time(nullptr);
+        if (s_tablet_report_failure_start_time.load() == 0) {
+            s_tablet_report_failure_start_time.store(current_time);
+        }
+        DorisMetrics::instance()->tablet_report_continuous_failure_duration_s->set_value(
+                current_time - s_tablet_report_failure_start_time);
         return;
     }
 
@@ -1227,6 +1235,15 @@ void report_tablet_callback(StorageEngine& engine, const ClusterInfo* cluster_in
     report_tablet_total << 1;
     if (!succ) [[unlikely]] {
         report_tablet_failed << 1;
+        int64_t current_time = time(nullptr);
+        if (s_tablet_report_failure_start_time.load() == 0) {
+            s_tablet_report_failure_start_time.store(current_time);
+        }
+        DorisMetrics::instance()->tablet_report_continuous_failure_duration_s->set_value(
+                current_time - s_tablet_report_failure_start_time);
+    } else {
+        s_tablet_report_failure_start_time.store(0);
+        DorisMetrics::instance()->tablet_report_continuous_failure_duration_s->set_value(0);
     }
 }
 
@@ -1254,9 +1271,16 @@ void report_tablet_callback(CloudStorageEngine& engine, const ClusterInfo* clust
         }
     }
 
-    if (report_version < s_report_version) {
+    if (report_version < s_report_version || UNLIKELY(config::enable_debug_points &&
+               DebugPoints::instance()->is_enable("WorkPoolCloudReportTablet.report_tablet_callback.skip"))) {
         LOG(WARNING) << "report version " << report_version << " change to " << s_report_version;
         DorisMetrics::instance()->report_all_tablets_requests_skip->increment(1);
+        int64_t current_time = time(nullptr);
+        if (s_tablet_report_failure_start_time.load() == 0) {
+            s_tablet_report_failure_start_time.store(current_time);
+        }
+        DorisMetrics::instance()->tablet_report_continuous_failure_duration_s->set_value(
+                current_time - s_tablet_report_failure_start_time);
         return;
     }
 
@@ -1267,6 +1291,15 @@ void report_tablet_callback(CloudStorageEngine& engine, const ClusterInfo* clust
     report_tablet_total << 1;
     if (!succ) [[unlikely]] {
         report_tablet_failed << 1;
+        int64_t current_time = time(nullptr);
+        if (s_tablet_report_failure_start_time.load() == 0) {
+            s_tablet_report_failure_start_time.store(current_time);
+        }
+        DorisMetrics::instance()->tablet_report_continuous_failure_duration_s->set_value(
+                current_time - s_tablet_report_failure_start_time);
+    } else {
+        s_tablet_report_failure_start_time.store(0);
+        DorisMetrics::instance()->tablet_report_continuous_failure_duration_s->set_value(0);
     }
 }
 

be/src/util/doris_metrics.cpp

@@ -250,6 +250,8 @@ DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(runtime_filter_consumer_timeout_num, Met
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(get_remote_tablet_slow_time_ms, MetricUnit::MILLISECONDS);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(get_remote_tablet_slow_cnt, MetricUnit::NOUNIT);
 
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_report_continuous_failure_duration_s, MetricUnit::SECONDS);
+
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(ann_index_load_costs_ms, MetricUnit::MILLISECONDS);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(ann_index_load_cnt, MetricUnit::NOUNIT);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(ann_index_search_costs_ms, MetricUnit::MILLISECONDS);
@@ -419,6 +421,8 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) {
     INT_COUNTER_METRIC_REGISTER(_server_metric_entity, get_remote_tablet_slow_time_ms);
     INT_COUNTER_METRIC_REGISTER(_server_metric_entity, get_remote_tablet_slow_cnt);
 
+    INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_report_continuous_failure_duration_s);
+
     INT_COUNTER_METRIC_REGISTER(_server_metric_entity, pipeline_task_queue_size);
 
     INT_COUNTER_METRIC_REGISTER(_server_metric_entity, ann_index_load_costs_ms);

be/src/util/doris_metrics.h

@@ -155,6 +155,9 @@ class DorisMetrics {
     IntGauge* tablet_cumulative_max_compaction_score = nullptr;
     IntGauge* tablet_base_max_compaction_score = nullptr;
 
+    // tablet report
+    IntGauge* tablet_report_continuous_failure_duration_s = nullptr;
+
     IntGauge* all_rowsets_num = nullptr;
     IntGauge* all_segments_num = nullptr;