From e6f8b0b0269a3e9dd5a82f7a9eceec10213a2d3b Mon Sep 17 00:00:00 2001 From: Fedor Sakharov Date: Thu, 18 Jan 2024 15:02:20 +0100 Subject: [PATCH] chore(withdrawal-finalizer): adds metrics for unfinalized eth withdrawals (#351) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `cargo fmt`. --- bin/withdrawal-finalizer/src/main.rs | 8 +++ bin/withdrawal-finalizer/src/metrics.rs | 36 ++++++++++ ...58786961de485d7f109005ee54d78f3fdd55e.json | 22 ++++++ ...a5bc149d100f740be557d170e7f1e3a055532.json | 20 ++++++ storage/src/lib.rs | 71 +++++++++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 storage/.sqlx/query-69b33bf2e4eeb6e6d37b8651fea58786961de485d7f109005ee54d78f3fdd55e.json create mode 100644 storage/.sqlx/query-a3dbb0522919f63706f3576b394a5bc149d100f740be557d170e7f1e3a055532.json diff --git a/bin/withdrawal-finalizer/src/main.rs b/bin/withdrawal-finalizer/src/main.rs index b5716480..daf2c5f1 100644 --- a/bin/withdrawal-finalizer/src/main.rs +++ b/bin/withdrawal-finalizer/src/main.rs @@ -304,6 +304,11 @@ async fn main() -> Result<()> { ); let finalizer_handle = tokio::spawn(finalizer.run(client_l2)); + let metrics_handle = tokio::spawn(metrics::meter_unfinalized_withdrawals( + pgpool.clone(), + eth_finalization_threshold, + )); + let api_server = tokio::spawn(api::run_server(pgpool)); tokio::select! { @@ -322,6 +327,9 @@ async fn main() -> Result<()> { r = finalizer_handle => { tracing::error!("Finalizer ended with {r:?}"); } + _ = metrics_handle => { + tracing::error!("Metrics loop has ended"); + } } stop_vise_exporter.send_replace(()); diff --git a/bin/withdrawal-finalizer/src/metrics.rs b/bin/withdrawal-finalizer/src/metrics.rs index e772420c..235dafec 100644 --- a/bin/withdrawal-finalizer/src/metrics.rs +++ b/bin/withdrawal-finalizer/src/metrics.rs @@ -1,7 +1,13 @@ //! Metrics for main binary +use std::time::Duration; + +use ethers::types::U256; +use sqlx::PgPool; use vise::{Gauge, Metrics}; +const METRICS_REFRESH_PERIOD: Duration = Duration::from_secs(15); + /// Main finalizer binary metrics #[derive(Debug, Metrics)] #[metrics(prefix = "withdrawal_finalizer")] @@ -11,7 +17,37 @@ pub(super) struct FinalizerMainMetrics { /// Capacity of the channel sending L2 events. pub watcher_l2_channel_capacity: Gauge, + + /// The withdrawals that were not finalized but are executed + pub executed_eth_withdrawals_not_finalized: Gauge, + + /// The withdrawals that + pub unexecuted_eth_withdrawals_below_current_threshold: Gauge, } #[vise::register] pub(super) static MAIN_FINALIZER_METRICS: vise::Global = vise::Global::new(); + +pub async fn meter_unfinalized_withdrawals(pool: PgPool, eth_threshold: Option) { + loop { + tokio::time::sleep(METRICS_REFRESH_PERIOD).await; + + let Ok(executed_not_finalized) = + storage::get_executed_and_not_finalized_withdrawals_count(&pool).await + else { + continue; + }; + let Ok(unexecuted) = storage::get_unexecuted_withdrawals_count(&pool, eth_threshold).await + else { + continue; + }; + + MAIN_FINALIZER_METRICS + .executed_eth_withdrawals_not_finalized + .set(executed_not_finalized); + + MAIN_FINALIZER_METRICS + .unexecuted_eth_withdrawals_below_current_threshold + .set(unexecuted); + } +} diff --git a/storage/.sqlx/query-69b33bf2e4eeb6e6d37b8651fea58786961de485d7f109005ee54d78f3fdd55e.json b/storage/.sqlx/query-69b33bf2e4eeb6e6d37b8651fea58786961de485d7f109005ee54d78f3fdd55e.json new file mode 100644 index 00000000..adcb8db3 --- /dev/null +++ b/storage/.sqlx/query-69b33bf2e4eeb6e6d37b8651fea58786961de485d7f109005ee54d78f3fdd55e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n COUNT(*)\n FROM\n finalization_data\n JOIN withdrawals w ON finalization_data.withdrawal_id = w.id\n WHERE\n finalization_tx IS NULL\n AND finalization_data.l2_block_number > COALESCE(\n (\n SELECT\n MAX(l2_block_number)\n FROM\n l2_blocks\n WHERE\n execute_l1_block_number IS NOT NULL\n ),\n 1\n )\n AND token = decode('000000000000000000000000000000000000800A', 'hex') \n AND amount >= $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Numeric" + ] + }, + "nullable": [ + null + ] + }, + "hash": "69b33bf2e4eeb6e6d37b8651fea58786961de485d7f109005ee54d78f3fdd55e" +} diff --git a/storage/.sqlx/query-a3dbb0522919f63706f3576b394a5bc149d100f740be557d170e7f1e3a055532.json b/storage/.sqlx/query-a3dbb0522919f63706f3576b394a5bc149d100f740be557d170e7f1e3a055532.json new file mode 100644 index 00000000..bbf830fe --- /dev/null +++ b/storage/.sqlx/query-a3dbb0522919f63706f3576b394a5bc149d100f740be557d170e7f1e3a055532.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n COUNT(*)\n FROM\n finalization_data\n JOIN withdrawals w ON finalization_data.withdrawal_id = w.id\n WHERE\n finalization_tx IS NULL\n AND failed_finalization_attempts = 0\n AND finalization_data.l2_block_number <= COALESCE(\n (\n SELECT\n MAX(l2_block_number)\n FROM\n l2_blocks\n WHERE\n execute_l1_block_number IS NOT NULL\n ),\n 1\n )\n AND token = decode('000000000000000000000000000000000000800A', 'hex') \n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "a3dbb0522919f63706f3576b394a5bc149d100f740be557d170e7f1e3a055532" +} diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 17a35af6..4f09c8eb 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -974,6 +974,77 @@ pub async fn withdrawals_to_finalize( Ok(data) } +/// Get the number of ETH withdrawals not yet executed and finalized and above some threshold +pub async fn get_unexecuted_withdrawals_count( + pool: &PgPool, + eth_threshold: Option, +) -> Result { + // if no threshold, query _all_ ethereum withdrawals since all of them are >= 0. + let eth_threshold = eth_threshold.unwrap_or(U256::zero()); + + let count = sqlx::query!( + " + SELECT + COUNT(*) + FROM + finalization_data + JOIN withdrawals w ON finalization_data.withdrawal_id = w.id + WHERE + finalization_tx IS NULL + AND finalization_data.l2_block_number > COALESCE( + ( + SELECT + MAX(l2_block_number) + FROM + l2_blocks + WHERE + execute_l1_block_number IS NOT NULL + ), + 1 + ) + AND token = decode('000000000000000000000000000000000000800A', 'hex') + AND amount >= $1 + ", + u256_to_big_decimal(eth_threshold), + ) + .fetch_one(pool) + .await?; + + Ok(count.count.unwrap_or(0)) +} + +/// Get the number of ETH withdrawals executed but not finalized +pub async fn get_executed_and_not_finalized_withdrawals_count(pool: &PgPool) -> Result { + let count = sqlx::query!( + " + SELECT + COUNT(*) + FROM + finalization_data + JOIN withdrawals w ON finalization_data.withdrawal_id = w.id + WHERE + finalization_tx IS NULL + AND failed_finalization_attempts = 0 + AND finalization_data.l2_block_number <= COALESCE( + ( + SELECT + MAX(l2_block_number) + FROM + l2_blocks + WHERE + execute_l1_block_number IS NOT NULL + ), + 1 + ) + AND token = decode('000000000000000000000000000000000000800A', 'hex') + ", + ) + .fetch_one(pool) + .await?; + + Ok(count.count.unwrap_or(0)) +} + /// Fetch finalization parameters for some withdrawal pub async fn get_finalize_withdrawal_params( pool: &PgPool,