diff --git a/.env.sample b/.env.sample index 330a91a..203caaf 100644 --- a/.env.sample +++ b/.env.sample @@ -1,6 +1,5 @@ # Database -DATABASE_URL=postgresql://postgres:rindexer@localhost:5440/postgres -POSTGRES_PASSWORD=rindexer +DATABASE_URL=rindexer.db API_PORT=3000 # Indexer & API diff --git a/Cargo.lock b/Cargo.lock index d0a5052..29bb73c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2002,7 +2002,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.4", + "webpki-roots", ] [[package]] @@ -2350,6 +2350,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ + "cc", "pkg-config", "vcpkg", ] @@ -3116,7 +3117,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.4", + "webpki-roots", ] [[package]] @@ -3595,15 +3596,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "signal-hook-registry" -version = "1.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" -dependencies = [ - "libc", -] - [[package]] name = "signature" version = "2.2.0" @@ -3679,7 +3671,6 @@ checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64", "bytes", - "chrono", "crc", "crossbeam-queue", "either", @@ -3695,7 +3686,6 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls", "serde", "serde_json", "sha2", @@ -3705,7 +3695,6 @@ dependencies = [ "tokio-stream", "tracing", "url", - "webpki-roots 0.26.11", ] [[package]] @@ -3757,7 +3746,6 @@ dependencies = [ "bitflags", "byteorder", "bytes", - "chrono", "crc", "digest 0.10.7", "dotenvy", @@ -3799,7 +3787,6 @@ dependencies = [ "base64", "bitflags", "byteorder", - "chrono", "crc", "dotenvy", "etcetera", @@ -3834,7 +3821,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ "atoi", - "chrono", "flume", "futures-channel", "futures-core", @@ -4132,9 +4118,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -4592,15 +4576,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.4", -] - [[package]] name = "webpki-roots" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index 20042f5..f08b9e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,8 @@ edition = "2024" [dependencies] axum = "0.7" -tokio = { version = "1.48", features = ["full"] } -sqlx = { version = "0.8.6", features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "json"] } +tokio = { version = "1.48", default-features = false } +sqlx = { version = "0.8.6", features = ["runtime-tokio", "json", "sqlite"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tower-http = { version = "0.5", features = ["cors"] } diff --git a/Dockerfile b/Dockerfile index 186300e..878d9e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN cargo build --release FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y ca-certificates libsqlite3-0 && rm -rf /var/lib/apt/lists/* COPY --from=builder /usr/src/app/target/release/subindexer /usr/local/bin/subindexer diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index c2ea73a..2c7c4fb 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -1,44 +1,24 @@ volumes: - postgres_data: + sqlite_data: driver: local services: - postgresql: - image: postgres:18 - shm_size: 1g - restart: always - volumes: - - postgres_data:/var/lib/postgresql - ports: - - 5440:5432 - env_file: - - ./.env - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 10s indexer: - image: ghcr.io/joshstevens19/rindexer - depends_on: - postgresql: - condition: service_healthy + image: ghcr.io/bh2smith/rindexer:sha-c0ea448eaabbff6a331c6538fae3016702e53817 volumes: - - .:/app/project_path - - /var/run/docker.sock:/var/run/docker.sock - working_dir: /app/project_path + - sqlite_data:/app/data + working_dir: /app + entrypoint: ["/app/rindexer"] command: [ "start", "indexer"] environment: - DATABASE_URL: postgres://postgres:rindexer@postgresql:5432/postgres + DATABASE_URL: /app/data/rindexer.db api: image: ghcr.io/deluxtreme/subindexer:latest - depends_on: - postgresql: - condition: service_healthy + volumes: + - sqlite_data:/app/data environment: - DATABASE_URL: postgres://postgres:rindexer@postgresql:5432/postgres + DATABASE_URL: /app/data/rindexer.db REDEEMER_PK: ${REDEEMER_PK} API_PORT: ${API_PORT:-3000} REDEEM_INTERVAL: ${REDEEM_INTERVAL:-3600} diff --git a/docker-compose.yml b/docker-compose.yml index 7b1f1b8..b19b562 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,34 +1,12 @@ -volumes: - postgres_data: - driver: local - services: - postgresql: - image: postgres:18 - shm_size: 1g - restart: always - volumes: - - postgres_data:/var/lib/postgresql - ports: - - 5440:5432 - environment: - POSTGRES_PASSWORD: rindexer - env_file: - - ./.env - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 10s indexer: - image: ghcr.io/joshstevens19/rindexer - depends_on: - postgresql: - condition: service_healthy + build: + context: . + dockerfile: Dockerfile.indexer volumes: - .:/app/project_path working_dir: /app/project_path - command: [ "start", "indexer"] + entrypoint: ["/app/rindexer"] + command: ["start", "indexer"] environment: - DATABASE_URL: postgres://postgres:rindexer@postgresql:5432/postgres + DATABASE_URL: rindexer.db diff --git a/rindexer.yaml.template b/rindexer.yaml.template index 1148671..ec8ad13 100644 --- a/rindexer.yaml.template +++ b/rindexer.yaml.template @@ -5,7 +5,7 @@ networks: chain_id: 100 rpc: $GNOSIS_RPC_URL storage: - postgres: + sqlite: enabled: true native_transfers: enabled: false diff --git a/src/api.rs b/src/api.rs index 569c5bc..87dd337 100644 --- a/src/api.rs +++ b/src/api.rs @@ -5,23 +5,16 @@ use axum::{ extract::{Query, State}, }; use serde::Deserialize; -use sqlx::PgPool; +use sqlx::SqlitePool; pub async fn get_redeemable( - State(pool): State, + State(pool): State, ) -> Json> { let current_timestamp = chrono::Utc::now().timestamp() as i32; match db::get_redeemable_subscriptions(&pool, current_timestamp).await { Ok(subscriptions) => Json(subscriptions), - Err(sqlx::Error::Database(db_err)) - if db_err.code() == Some(std::borrow::Cow::Borrowed("42P01")) => - { - // Table doesn't exist yet, return empty list - tracing::warn!("Database tables don't exist yet, returning empty list"); - Json(Vec::new()) - } Err(e) => { - // Log other database errors but don't panic + // Log database errors but don't panic (tables might not exist yet) tracing::error!("Database error: {}", e); Json(Vec::new()) } @@ -35,7 +28,7 @@ pub struct SubscriptionsQuery { } pub async fn get_subscriptions( - State(pool): State, + State(pool): State, Query(query): Query, ) -> Result>, (axum::http::StatusCode, Json)> { match db::get_user_subscriptions(&pool, query.subscriber, query.recipient).await { @@ -51,7 +44,7 @@ pub async fn get_subscriptions( } } -pub async fn health_check(State(pool): State) -> Json { +pub async fn health_check(State(pool): State) -> Json { // Test database connectivity let db_healthy = (sqlx::query("SELECT 1").execute(&pool).await).is_ok(); let blocks_behind = db::check_liveness(&pool).await.unwrap_or(u64::MAX); diff --git a/src/config.rs b/src/config.rs index 5fd2629..d9cc7e9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,13 +1,13 @@ use std::{env, str::FromStr}; use alloy::signers::local::PrivateKeySigner; -use sqlx::{PgPool, postgres::PgPoolOptions}; +use sqlx::{SqlitePool, sqlite::SqlitePoolOptions}; // One hour on (gnosis chain). pub const STALE_BLOCK_THRESHOLD: u64 = 12 * 60; pub struct Config { - pub pool: PgPool, + pub pool: SqlitePool, pub redeemer: Option, // Optional overrides: pub api_port: u16, @@ -18,11 +18,12 @@ pub struct Config { impl Config { pub async fn from_env() -> Self { let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let pool = PgPoolOptions::new() + tracing::info!("Connecting to SQLite database at {}", database_url); + let pool = SqlitePoolOptions::new() .max_connections(2) .connect(&database_url) .await - .expect("Failed to connect to Postgres"); + .expect("Failed to connect to SQLite"); let redeemer_pk = env::var("REDEEMER_PK").ok(); Self { pool, diff --git a/src/db.rs b/src/db.rs index 2b8d806..4e1f1c4 100644 --- a/src/db.rs +++ b/src/db.rs @@ -3,13 +3,13 @@ use alloy::{ primitives::Address, providers::{Provider, ProviderBuilder}, }; -use anyhow::Result; -use sqlx::PgPool; +use anyhow::{Context, Result}; +use sqlx::SqlitePool; const REDEEMABLE_QUERY: &str = include_str!("queries/redeemable.sql"); pub async fn get_redeemable_subscriptions( - pool: &PgPool, + pool: &SqlitePool, current_timestamp: i32, ) -> Result, sqlx::Error> { sqlx::query_as::<_, RedeemableSubscription>(REDEEMABLE_QUERY) @@ -18,9 +18,9 @@ pub async fn get_redeemable_subscriptions( .await } -async fn get_last_synced_block(pool: &PgPool) -> Result { +async fn get_last_synced_block(pool: &SqlitePool) -> Result { sqlx::query_scalar::<_, i64>( - "SELECT block::bigint FROM rindexer_internal.latest_block WHERE network = 'gnosis'", + "SELECT block FROM rindexer_internal_latest_block WHERE network = 'gnosis'", ) .fetch_one(pool) .await @@ -28,8 +28,10 @@ async fn get_last_synced_block(pool: &PgPool) -> Result { } // Returns number of blocks behind latest -pub async fn check_liveness(pool: &PgPool) -> Result { - let last_synced_block = get_last_synced_block(pool).await?; +pub async fn check_liveness(pool: &SqlitePool) -> Result { + let last_synced_block = get_last_synced_block(pool) + .await + .context("Failed to get last synced block")?; // Use a different RPC as indexer (because node may not be synced.) let provider = ProviderBuilder::new().connect_http("https://rpc.gnosischain.com/".parse()?); @@ -40,7 +42,7 @@ pub async fn check_liveness(pool: &PgPool) -> Result { const USER_SUBSCRIPTIONS_QUERY: &str = include_str!("queries/user_subscriptions.sql"); pub async fn get_user_subscriptions( - pool: &PgPool, + pool: &SqlitePool, subscriber: Option
, recipient: Option
, ) -> Result, sqlx::Error> { diff --git a/src/models.rs b/src/models.rs index 2f4dce6..3154cf3 100644 --- a/src/models.rs +++ b/src/models.rs @@ -4,8 +4,7 @@ use sqlx::FromRow; #[derive(Debug, Serialize, Deserialize, FromRow)] pub struct RedeemableSubscription { pub contract_address: String, - #[serde(serialize_with = "hex::serialize")] - pub id: Vec, + pub id: String, pub subscriber: String, pub recipient: String, pub amount: String, @@ -17,8 +16,7 @@ pub struct RedeemableSubscription { #[derive(Debug, Serialize, Deserialize, FromRow)] pub struct Subscription { pub contract_address: String, - #[serde(serialize_with = "hex::serialize")] - pub id: Vec, + pub id: String, pub subscriber: String, pub recipient: String, pub amount: String, @@ -27,22 +25,36 @@ pub struct Subscription { pub creation_timestamp: i32, } -#[derive(Debug, Serialize, Deserialize, sqlx::Type, PartialEq)] -#[repr(i16)] +#[derive(Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "lowercase")] pub enum Category { - Trusted = 0, - Untrusted = 1, - Group = 2, + Trusted, + Untrusted, + Group, } -mod hex { - use serde::Serializer; +impl sqlx::Type for Category +where + String: sqlx::Type, +{ + fn type_info() -> ::TypeInfo { + >::type_info() + } +} - pub fn serialize(bytes: &Vec, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&format!("0x{}", alloy::hex::encode(bytes))) +impl<'r> sqlx::Decode<'r, sqlx::Sqlite> for Category { + fn decode( + value: ::ValueRef<'r>, + ) -> Result { + let value = >::decode(value)?; + match value.as_str() { + "0" => Ok(Category::Trusted), + "1" => Ok(Category::Untrusted), + "2" => Ok(Category::Group), + "trusted" => Ok(Category::Trusted), + "untrusted" => Ok(Category::Untrusted), + "group" => Ok(Category::Group), + _ => Err(format!("invalid category value: {}", value).into()), + } } } diff --git a/src/queries/redeemable.sql b/src/queries/redeemable.sql index 584c266..1039f73 100644 --- a/src/queries/redeemable.sql +++ b/src/queries/redeemable.sql @@ -7,27 +7,39 @@ WITH recipient, amount, category, - frequency::INTEGER as frequency, - FLOOR(EXTRACT(EPOCH FROM now())) as right_meow, - creation_timestamp::INTEGER as creation_timestamp - FROM subindexer_subscription_module.subscription_created active - LEFT JOIN subindexer_subscription_module.unsubscribed canceled + frequency, + CAST(strftime('%s', 'now') AS INTEGER) as right_meow, + creation_timestamp + FROM subindexer_subscription_module_subscription_created active + LEFT JOIN subindexer_subscription_module_unsubscribed canceled ON active.id = canceled.id WHERE canceled.id IS NULL ), latest_redemptions AS ( - SELECT DISTINCT ON (id) + SELECT id, - next_redeem_at::INTEGER as next_redeem_at - FROM subindexer_subscription_module.redeemed - ORDER BY id, next_redeem_at DESC + next_redeem_at + FROM ( + SELECT + id, + next_redeem_at, + ROW_NUMBER() OVER (PARTITION BY id ORDER BY next_redeem_at DESC) as rn + FROM subindexer_subscription_module_redeemed + ) + WHERE rn = 1 ), latest_recipients AS ( - SELECT DISTINCT ON (id) + SELECT id, new_recipient - FROM subindexer_subscription_module.recipient_updated - ORDER BY id, block_number DESC + FROM ( + SELECT + id, + new_recipient, + ROW_NUMBER() OVER (PARTITION BY id ORDER BY block_number DESC) as rn + FROM subindexer_subscription_module_recipient_updated + ) + WHERE rn = 1 ), upcoming AS ( SELECT @@ -37,9 +49,9 @@ WITH COALESCE(rp.new_recipient, a.recipient) AS recipient, amount, -- Redeemable Periods: cf https://github.com/deluXtreme/subi-contracts/blob/65455f02e3e7a49654c51b9b5e805cccc1032168/src/SubscriptionModule.sol#L154-L158 - FLOOR((right_meow - COALESCE(r.next_redeem_at, creation_timestamp) + frequency) / a.frequency)::INTEGER as periods, + CAST((right_meow - COALESCE(CAST(r.next_redeem_at AS INTEGER), CAST(creation_timestamp AS INTEGER)) + CAST(frequency AS INTEGER)) / CAST(a.frequency AS INTEGER) AS INTEGER) as periods, category, - COALESCE(r.next_redeem_at, creation_timestamp) AS next_redeem_at + CAST(COALESCE(r.next_redeem_at, creation_timestamp) AS INTEGER) AS next_redeem_at FROM active_subscriptions a LEFT JOIN latest_redemptions r ON a.id = r.id diff --git a/src/queries/user_subscriptions.sql b/src/queries/user_subscriptions.sql index 4322aab..3e1331a 100644 --- a/src/queries/user_subscriptions.sql +++ b/src/queries/user_subscriptions.sql @@ -5,12 +5,12 @@ SELECT recipient, amount, category, - frequency::INTEGER as frequency, - creation_timestamp::INTEGER as creation_timestamp -FROM subindexer_subscription_module.subscription_created active - LEFT JOIN subindexer_subscription_module.unsubscribed canceled + CAST(frequency AS INTEGER) as frequency, + CAST(creation_timestamp AS INTEGER) as creation_timestamp +FROM subindexer_subscription_module_subscription_created active + LEFT JOIN subindexer_subscription_module_unsubscribed canceled ON active.id = canceled.id WHERE canceled.id IS NULL -AND ($1::text IS NULL OR active.subscriber = $1) -AND ($2::text IS NULL OR active.recipient = $2); +AND ($1 IS NULL OR active.subscriber = $1) +AND ($2 IS NULL OR active.recipient = $2); diff --git a/src/redeem.rs b/src/redeem.rs index 3f93ca2..bf5b6b5 100644 --- a/src/redeem.rs +++ b/src/redeem.rs @@ -7,7 +7,7 @@ use alloy::{ sol, }; use anyhow::Result; -use sqlx::PgPool; +use sqlx::SqlitePool; use circles_pathfinder::{FindPathParams, encode_redeem_trusted_data, prepare_flow_for_contract}; use std::str::FromStr; @@ -22,7 +22,11 @@ use crate::{ const EXPLORER_URL: &str = "https://gnosisscan.io/tx"; -pub async fn run_redeem_job(rpc_url: &str, pool: &PgPool, signer: &PrivateKeySigner) -> Result<()> { +pub async fn run_redeem_job( + rpc_url: &str, + pool: &SqlitePool, + signer: &PrivateKeySigner, +) -> Result<()> { tracing::info!("Running redeem job with signer: {:?}", signer.address()); // Ensure indexer liveness. let blocks_behind = db::check_liveness(pool).await?; @@ -116,7 +120,7 @@ async fn encode_tx( contract: SubscriptionModuleInstance

, subscription: &RedeemableSubscription, ) -> Result { - let id = U256::from_be_slice(&subscription.id); + let id = U256::from_str_radix(subscription.id.trim_start_matches("0x"), 16)?; let tx; if subscription.category != Category::Trusted { tx = contract.redeem(id.into(), vec![].into());