From 3661ca225a11de77d259ac95d16fe914d71d84ec Mon Sep 17 00:00:00 2001 From: simzzz Date: Fri, 28 May 2021 18:36:05 +0300 Subject: [PATCH 01/49] committing current progress --- sentry/src/db/campaign.rs | 50 +++++++++++++ sentry/src/lib.rs | 11 +++ sentry/src/routes/campaign.rs | 137 +++++++++++++++++++++++++++++----- 3 files changed, 178 insertions(+), 20 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index c3660d68e..edab437a4 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -49,6 +49,51 @@ pub async fn fetch_campaign(pool: DbPool, campaign: &Campaign) -> Result Result, PoolError> { + let client = pool.get().await?; + let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE channel_id = $1").await?; + + let row = client.query(&statement, &[&campaign.channel.id()]).await?; + + let campaigns = row.into_iter().for_each(|c| Campaign::from(c)).collect(); + Ok(campaigns); +} + +pub async fn campaign_exists(pool: &DbPool, campaign: &Campaign) -> Result { + let client = pool.get().await?; + let statement = client + .prepare("SELECT EXISTS(SELECT 1 FROM campaigns WHERE id = $1)") + .await?; + + let row = client.execute(&statement, &[&campaign.id]).await?; + + let exists = row == 1; + Ok(exists) +} + +pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result { + let client = pool.get().await?; + let statement = client + .prepare("UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 WHERE id = $8") + .await?; + + let row = client + .execute(&statement, &[ + &campaign.budget, + &campaign.validators, + &campaign.title, + &campaign.pricing_bounds, + &campaign.event_submission, + &campaign.ad_units, + &campaign.targeting_rules, + &campaign.id, + ]) + .await?; + + let exists = row == 1; + Ok(exists) +} + #[cfg(test)] mod test { use primitives::{ @@ -77,6 +122,11 @@ mod test { assert!(is_inserted); + let exists = campaign_exists(&db_pool.clone(), campaign: &campaign_for_testing) + .await + .expect("Should succeed"); + asser!(exists); + let fetched_campaign: Campaign = fetch_campaign(db_pool.clone(), &campaign_for_testing) .await .expect("Should fetch successfully"); diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 403df405a..455d8833c 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -21,6 +21,7 @@ use primitives::{Config, ValidatorId}; use redis::aio::MultiplexedConnection; use regex::Regex; use routes::analytics::{advanced_analytics, advertiser_analytics, analytics, publisher_analytics}; +use routes::campaign::create_campaign; use routes::cfg::config; use routes::channel::{ channel_list, channel_validate, create_channel, create_validator_messages, insert_events, @@ -150,6 +151,16 @@ impl Application { publisher_analytics(req, &self).await } + ("/campaign.create", &Method::POST) => { + let req = match AuthRequired.call(req, &self).await { + Ok(req) => req, + Err(error) => { + return map_response_error(error); + } + }; + + create_campaign(req, &self).await + } (route, _) if route.starts_with("/analytics") => analytics_router(req, &self).await, // This is important becuase it prevents us from doing // expensive regex matching for routes without /channel diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 94076e34b..58747bfad 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,7 +1,21 @@ -use crate::{success_response, Application, Auth, ResponseError, RouteParams, Session}; +use crate::{ + success_response, Application, Auth, ResponseError, RouteParams, Session, + event_aggregate::latest_new_state, + db::{ + spendable::fetch_spendable, + campaign::{campaign_exists, update_campaign, insert_campaign, get_campaigns_for_channel}, + } +}; use hyper::{Body, Request, Response}; -use primitives::{adapter::Adapter, sentry::{ - campaign_create::CreateCampaign,SuccessResponse}}; +use primitives::{ + adapter::Adapter, + sentry::{ + campaign_create::CreateCampaign, + SuccessResponse + }, + Campaign +}; +use redis::aio::MultiplexedConnection; pub async fn create_campaign( req: Request, @@ -20,25 +34,108 @@ pub async fn create_campaign( let error_response = ResponseError::BadRequest("err occurred; please try again later".into()); // insert Campaign - - // match insert_campaign(&app.pool, &campaign).await { - // Err(error) => { - // error!(&app.logger, "{}", &error; "module" => "create_channel"); - - // match error { - // PoolError::Backend(error) if error.code() == Some(&SqlState::UNIQUE_VIOLATION) => { - // Err(ResponseError::Conflict( - // "channel already exists".to_string(), - // )) - // } - // _ => Err(error_response), - // } - // } - // Ok(false) => Err(error_response), - // _ => Ok(()), - // }?; + + match insert_or_modify_campaign(&app.pool, &campaign, &app.redis).await { + Err(error) => { + error!(&app.logger, "{}", &error; "module" => "create_channel"); + + match error { + PoolError::Backend(error) if error.code() == Some(&SqlState::UNIQUE_VIOLATION) => { + Err(ResponseError::Conflict( + "channel already exists".to_string(), + )) + } + _ => Err(error_response), + } + } + Ok(false) => Err(error_response), + _ => Ok(()), + }?; let create_response = SuccessResponse { success: true }; Ok(success_response(serde_json::to_string(&campaign)?)) } + +// TODO: Double check redis calls +async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { + let key = format!("adexCampaign:campaignSpent:{}", id) + // campaignSpent tracks the portion of the budget which has already been spent + let campaign_spent = match redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await? + { + Some(spent) => UnifiedNum::from(spent), + // TODO: Double check if this is true + // If the campaign is just being inserted, there would be no entry therefore no funds would be spent + None => 0 + }; +} + +async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + // update a key in Redis for the remaining spendable amount + let key = format!("adexCampaign:remainingSpendable:{}", id) + redis::cmd("SET") + .arg(&key) + .arg(amount) + .query_async(&mut redis.clone()) + .await? +} + +async fn update_remaining_for_channel(redis: &MultiplexedConnection, id: ChannelId, amount: UnifiedNum) -> Result { + let key = format!("adexChannel:remaining:{}", id) + redis::cmd("SET") + .arg(&key) + .arg(amount) + .query_async(&mut redis.clone()) + .await? +} + +async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaign: &Campaign) -> Result { + let campaigns_for_channel = get_campaigns_for_channel(&campaign).await?; + let sum_of_campaigns_remaining = campaigns_for_channel + .map(|c| { + let spent = get_spent_for_campaign(&redis, c.id).await?; + let remaining = c.budget - spent; + remaining + }) + .sum(); +} + +pub async fn insert_or_modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { + let campaign_spent = get_spent_for_campaign(&redis, campaign.id()).await?; + + // Check if we haven't exceeded the budget yet + if (campaign.budget <= campaign_spent) { + ResponseError::FailedValidation("No more budget available for spending") + } + + let remaining_spendable_campaign = campaign.budget - campaign_spent; + update_remaining_for_campaign(&redis, campaign.id, remaining_spendable_campaign).await?; + + + // Getting the latest new state from Postgres + let latest_new_state = latest_new_state(&pool, &campaign.channel, "").await?; + // Gets the latest Spendable for this (spender, channelId) pair + let latest_spendable = fetch_spendable(pool, campaign.creator, campaign.channel.id()).await?; + + let total_deposited = latest_spendable.deposit.total; + let total_spent = latest_new_state.spenders[campaign.creator]; + let total_remaining = total_deposited - total_spent; + + update_remaining_for_channel(&redis, campaign.channel.id(), total_remaining).await?; + + if (campaign_exists(&pool, &campaign)) { + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaign).await?; + if campaigns_remaining_sum > total_remaining { + ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel") + } + update_campaign(&pool, &campaign).await? + } + insert_campaign(&pool, &campaign).await? + + // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent + // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] + // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 +} From 8c4d68721ac8da55cd1295d71d80b7c6d3a99594 Mon Sep 17 00:00:00 2001 From: simzzz Date: Mon, 31 May 2021 12:32:04 +0300 Subject: [PATCH 02/49] fixed some more errors --- sentry/src/db.rs | 2 +- sentry/src/db/campaign.rs | 2 +- sentry/src/lib.rs | 1 + sentry/src/routes/campaign.rs | 36 ++++++++++++++++------------------- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/sentry/src/db.rs b/sentry/src/db.rs index 5fb148aeb..12d631afa 100644 --- a/sentry/src/db.rs +++ b/sentry/src/db.rs @@ -6,7 +6,7 @@ use tokio_postgres::NoTls; use lazy_static::lazy_static; pub mod analytics; -mod campaign; +pub mod campaign; mod channel; pub mod event_aggregate; pub mod spendable; diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 4002bb7bc..0df31ba16 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -56,7 +56,7 @@ pub async fn get_campaigns_for_channel(pool: DbPool, campaign: &Campaign) -> Res let row = client.query(&statement, &[&campaign.channel.id()]).await?; let campaigns = row.into_iter().for_each(|c| Campaign::from(c)).collect(); - Ok(campaigns); + Ok(campaigns) } pub async fn campaign_exists(pool: &DbPool, campaign: &Campaign) -> Result { diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 455d8833c..226644902 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -35,6 +35,7 @@ pub mod routes { pub mod analytics; pub mod cfg; pub mod channel; + pub mod campaign; pub mod event_aggregate; pub mod validator_message; } diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 58747bfad..626ddba7b 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,10 +1,10 @@ use crate::{ success_response, Application, Auth, ResponseError, RouteParams, Session, - event_aggregate::latest_new_state, db::{ spendable::fetch_spendable, - campaign::{campaign_exists, update_campaign, insert_campaign, get_campaigns_for_channel}, - } + event_aggregate::latest_new_state, + DbPool + }, }; use hyper::{Body, Request, Response}; use primitives::{ @@ -13,9 +13,13 @@ use primitives::{ campaign_create::CreateCampaign, SuccessResponse }, - Campaign + Campaign, CampaignId, UnifiedNum, ChannelId }; use redis::aio::MultiplexedConnection; +use deadpool_postgres::PoolError; +use slog::error; +use tokio_postgres::error::SqlState; +use crate::db::campaign::{campaign_exists, update_campaign, insert_campaign, get_campaigns_for_channel}; pub async fn create_campaign( req: Request, @@ -37,16 +41,8 @@ pub async fn create_campaign( match insert_or_modify_campaign(&app.pool, &campaign, &app.redis).await { Err(error) => { - error!(&app.logger, "{}", &error; "module" => "create_channel"); - - match error { - PoolError::Backend(error) if error.code() == Some(&SqlState::UNIQUE_VIOLATION) => { - Err(ResponseError::Conflict( - "channel already exists".to_string(), - )) - } - _ => Err(error_response), - } + // error!(&app.logger, "{}", &error; "module" => "create_channel"); + Err(ResponseError::Conflict("channel already exists".to_string())) } Ok(false) => Err(error_response), _ => Ok(()), @@ -59,7 +55,7 @@ pub async fn create_campaign( // TODO: Double check redis calls async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { - let key = format!("adexCampaign:campaignSpent:{}", id) + let key = format!("adexCampaign:campaignSpent:{}", id); // campaignSpent tracks the portion of the budget which has already been spent let campaign_spent = match redis::cmd("GET") .arg(&key) @@ -75,7 +71,7 @@ async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) - async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { // update a key in Redis for the remaining spendable amount - let key = format!("adexCampaign:remainingSpendable:{}", id) + let key = format!("adexCampaign:remainingSpendable:{}", id); redis::cmd("SET") .arg(&key) .arg(amount) @@ -84,7 +80,7 @@ async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: Campai } async fn update_remaining_for_channel(redis: &MultiplexedConnection, id: ChannelId, amount: UnifiedNum) -> Result { - let key = format!("adexChannel:remaining:{}", id) + let key = format!("adexChannel:remaining:{}", id); redis::cmd("SET") .arg(&key) .arg(amount) @@ -104,11 +100,11 @@ async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaign: &C } pub async fn insert_or_modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { - let campaign_spent = get_spent_for_campaign(&redis, campaign.id()).await?; + let campaign_spent = get_spent_for_campaign(&redis, campaign.id).await?; // Check if we haven't exceeded the budget yet if (campaign.budget <= campaign_spent) { - ResponseError::FailedValidation("No more budget available for spending") + ResponseError::FailedValidation("No more budget available for spending".into()) } let remaining_spendable_campaign = campaign.budget - campaign_spent; @@ -118,7 +114,7 @@ pub async fn insert_or_modify_campaign(pool: &DbPool, campaign: &Campaign, redis // Getting the latest new state from Postgres let latest_new_state = latest_new_state(&pool, &campaign.channel, "").await?; // Gets the latest Spendable for this (spender, channelId) pair - let latest_spendable = fetch_spendable(pool, campaign.creator, campaign.channel.id()).await?; + let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &*campaign.channel.id()).await?; let total_deposited = latest_spendable.deposit.total; let total_spent = latest_new_state.spenders[campaign.creator]; From 58c61aa79b43b1eab728461bed0e135545cc8227 Mon Sep 17 00:00:00 2001 From: simzzz Date: Mon, 31 May 2021 13:52:18 +0300 Subject: [PATCH 03/49] more error fixes --- sentry/src/db/campaign.rs | 4 +-- sentry/src/db/event_aggregate.rs | 27 ++++++++++++++++- sentry/src/routes/campaign.rs | 52 +++++++++++++++++++------------- 3 files changed, 59 insertions(+), 24 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 0df31ba16..5ba83e882 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -49,13 +49,13 @@ pub async fn fetch_campaign(pool: DbPool, campaign: &Campaign) -> Result Result, PoolError> { +pub async fn get_campaigns_for_channel(pool: &DbPool, campaign: &Campaign) -> Result, PoolError> { let client = pool.get().await?; let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE channel_id = $1").await?; let row = client.query(&statement, &[&campaign.channel.id()]).await?; - let campaigns = row.into_iter().for_each(|c| Campaign::from(c)).collect(); + let campaigns = row.into_iter().map(|c| Campaign::from(c)).collect(); Ok(campaigns) } diff --git a/sentry/src/db/event_aggregate.rs b/sentry/src/db/event_aggregate.rs index 73b73d138..c74c6cda9 100644 --- a/sentry/src/db/event_aggregate.rs +++ b/sentry/src/db/event_aggregate.rs @@ -3,7 +3,7 @@ use futures::pin_mut; use primitives::{ sentry::{EventAggregate, MessageResponse}, validator::{ApproveState, Heartbeat, NewState}, - Address, BigNum, Channel, ChannelId, ValidatorId, + Address, BigNum, Channel, ChannelId, ValidatorId, channel_v5::Channel as ChannelV5, }; use std::{convert::TryFrom, ops::Add}; use tokio_postgres::{ @@ -58,6 +58,31 @@ pub async fn latest_new_state( .map_err(PoolError::Backend) } +pub async fn latest_new_state_v5( + pool: &DbPool, + channel: &ChannelV5, + state_root: &str, +) -> Result>, PoolError> { + let client = pool.get().await?; + + let select = client.prepare("SELECT \"from\", msg, received FROM validator_messages WHERE channel_id = $1 AND \"from\" = $2 AND msg ->> 'type' = 'NewState' AND msg->> 'stateRoot' = $3 ORDER BY received DESC LIMIT 1").await?; + let rows = client + .query( + &select, + &[ + &channel.id(), + &channel.leader, + &state_root, + ], + ) + .await?; + + rows.get(0) + .map(MessageResponse::::try_from) + .transpose() + .map_err(PoolError::Backend) +} + pub async fn latest_heartbeats( pool: &DbPool, channel_id: &ChannelId, diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 626ddba7b..4ae2741e4 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -2,7 +2,7 @@ use crate::{ success_response, Application, Auth, ResponseError, RouteParams, Session, db::{ spendable::fetch_spendable, - event_aggregate::latest_new_state, + event_aggregate::latest_new_state_v5, DbPool }, }; @@ -42,7 +42,7 @@ pub async fn create_campaign( match insert_or_modify_campaign(&app.pool, &campaign, &app.redis).await { Err(error) => { // error!(&app.logger, "{}", &error; "module" => "create_channel"); - Err(ResponseError::Conflict("channel already exists".to_string())) + return Err(ResponseError::Conflict("channel already exists".to_string())); } Ok(false) => Err(error_response), _ => Ok(()), @@ -65,8 +65,9 @@ async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) - Some(spent) => UnifiedNum::from(spent), // TODO: Double check if this is true // If the campaign is just being inserted, there would be no entry therefore no funds would be spent - None => 0 + None => UnifiedNum::from(0) }; + Ok(campaign_spent) } async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { @@ -74,37 +75,41 @@ async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: Campai let key = format!("adexCampaign:remainingSpendable:{}", id); redis::cmd("SET") .arg(&key) - .arg(amount) + .arg(amount.to_u64()) .query_async(&mut redis.clone()) - .await? + .await?; + Ok(true) } async fn update_remaining_for_channel(redis: &MultiplexedConnection, id: ChannelId, amount: UnifiedNum) -> Result { let key = format!("adexChannel:remaining:{}", id); redis::cmd("SET") .arg(&key) - .arg(amount) + .arg(amount.to_u64()) .query_async(&mut redis.clone()) - .await? + .await?; + Ok(true) } -async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaign: &Campaign) -> Result { - let campaigns_for_channel = get_campaigns_for_channel(&campaign).await?; - let sum_of_campaigns_remaining = campaigns_for_channel - .map(|c| { +async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign) -> Result { + let campaigns_for_channel = get_campaigns_for_channel(&pool, &campaign).await?; + let sum_of_campaigns_remaining = campaigns_for_channel + .into_iter() + .map(async |c| { let spent = get_spent_for_campaign(&redis, c.id).await?; let remaining = c.budget - spent; remaining }) .sum(); + Ok(sum_of_campaigns_remaining) } pub async fn insert_or_modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { let campaign_spent = get_spent_for_campaign(&redis, campaign.id).await?; // Check if we haven't exceeded the budget yet - if (campaign.budget <= campaign_spent) { - ResponseError::FailedValidation("No more budget available for spending".into()) + if campaign.budget <= campaign_spent { + return Err(ResponseError::FailedValidation("No more budget available for spending".into())); } let remaining_spendable_campaign = campaign.budget - campaign_spent; @@ -112,24 +117,29 @@ pub async fn insert_or_modify_campaign(pool: &DbPool, campaign: &Campaign, redis // Getting the latest new state from Postgres - let latest_new_state = latest_new_state(&pool, &campaign.channel, "").await?; + let latest_new_state = latest_new_state_v5(&pool, &campaign.channel, "").await?; // Gets the latest Spendable for this (spender, channelId) pair - let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &*campaign.channel.id()).await?; + let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; let total_deposited = latest_spendable.deposit.total; - let total_spent = latest_new_state.spenders[campaign.creator]; + let total_spent = if let Some(lns) = latest_new_state { + lns.msg.into_inner().spenders[campaign.creator] + } else { + 0 + }; + let total_remaining = total_deposited - total_spent; update_remaining_for_channel(&redis, campaign.channel.id(), total_remaining).await?; - if (campaign_exists(&pool, &campaign)) { - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaign).await?; + if campaign_exists(&pool, &campaign).await? { + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &pool, &campaign).await?; if campaigns_remaining_sum > total_remaining { - ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel") + return Err(ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel".into())); } - update_campaign(&pool, &campaign).await? + return update_campaign(&pool, &campaign).await? } - insert_campaign(&pool, &campaign).await? + return insert_campaign(&pool, &campaign).await?; // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] From 70ff87c198210120ebcffe1a3445f10c698930af Mon Sep 17 00:00:00 2001 From: simzzz Date: Tue, 8 Jun 2021 19:12:05 +0300 Subject: [PATCH 04/49] Progress w/out tests --- primitives/src/sentry.rs | 12 +++ sentry/src/db/campaign.rs | 12 +-- sentry/src/lib.rs | 16 ++- sentry/src/routes/campaign.rs | 177 +++++++++++++++++++++++----------- 4 files changed, 149 insertions(+), 68 deletions(-) diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index da6bbacf2..166f1a117 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -379,6 +379,18 @@ pub mod campaign_create { } } } + + // All editable fields stored in one place, used for checking when a budget is changed + // #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] + // pub struct CampaignState { + // pub budget: UnifiedNum, + // pub validators: Validators, + // pub title: Option, + // pub pricing_bounds: Option, + // pub event_submission: Option, + // pub ad_units: Vec, + // pub targeting_rules: Rules, + // } } #[cfg(feature = "postgres")] diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 5ba83e882..62c2dd8fb 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -2,8 +2,6 @@ use crate::db::{DbPool, PoolError}; use primitives::Campaign; use tokio_postgres::types::Json; -// TODO: Remove once we use this fn -#[allow(dead_code)] pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let ad_units = Json(campaign.ad_units.clone()); @@ -38,9 +36,7 @@ pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result Result { +pub async fn fetch_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE id = $1").await?; @@ -55,7 +51,7 @@ pub async fn get_campaigns_for_channel(pool: &DbPool, campaign: &Campaign) -> Re let row = client.query(&statement, &[&campaign.channel.id()]).await?; - let campaigns = row.into_iter().map(|c| Campaign::from(c)).collect(); + let campaigns = row.into_iter().map(|c| Campaign::from(&c)).collect(); Ok(campaigns) } @@ -71,7 +67,7 @@ pub async fn campaign_exists(pool: &DbPool, campaign: &Campaign) -> Result Result { +pub async fn update_campaign_in_db(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let statement = client .prepare("UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 WHERE id = $8") @@ -122,7 +118,7 @@ mod test { .expect("Should succeed"); asser!(exists); - let fetched_campaign: Campaign = fetch_campaign(db_pool.clone(), &campaign_for_testing) + let fetched_campaign: Campaign = fetch_campaign(&db_pool, &campaign_for_testing) .await .expect("Should fetch successfully"); diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 226644902..993e1eed4 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -21,7 +21,7 @@ use primitives::{Config, ValidatorId}; use redis::aio::MultiplexedConnection; use regex::Regex; use routes::analytics::{advanced_analytics, advertiser_analytics, analytics, publisher_analytics}; -use routes::campaign::create_campaign; +use routes::campaign::{create_campaign, update_campaign}; use routes::cfg::config; use routes::channel::{ channel_list, channel_validate, create_channel, create_validator_messages, insert_events, @@ -152,7 +152,8 @@ impl Application { publisher_analytics(req, &self).await } - ("/campaign.create", &Method::POST) => { + // For creating campaigns + ("/campaign", &Method::POST) => { let req = match AuthRequired.call(req, &self).await { Ok(req) => req, Err(error) => { @@ -162,6 +163,17 @@ impl Application { create_campaign(req, &self).await } + // For editing campaigns + ("/campaign/:id", &Method::POST) => { + let req = match AuthRequired.call(req, &self).await { + Ok(req) => req, + Err(error) => { + return map_response_error(error); + } + }; + + update_campaign(req, &self).await + } (route, _) if route.starts_with("/analytics") => analytics_router(req, &self).await, // This is important becuase it prevents us from doing // expensive regex matching for routes without /channel diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 4ae2741e4..415e38aff 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -13,13 +13,14 @@ use primitives::{ campaign_create::CreateCampaign, SuccessResponse }, - Campaign, CampaignId, UnifiedNum, ChannelId + Campaign, CampaignId, UnifiedNum, ChannelId, BigNum }; use redis::aio::MultiplexedConnection; use deadpool_postgres::PoolError; use slog::error; use tokio_postgres::error::SqlState; -use crate::db::campaign::{campaign_exists, update_campaign, insert_campaign, get_campaigns_for_channel}; +use crate::db::campaign::{campaign_exists, update_campaign_in_db, insert_campaign, get_campaigns_for_channel, fetch_campaign}; +use std::{convert::TryFrom, fs, str::FromStr}; pub async fn create_campaign( req: Request, @@ -35,14 +36,23 @@ pub async fn create_campaign( // TODO AIP#61: Validate Campaign - let error_response = ResponseError::BadRequest("err occurred; please try again later".into()); + let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); - // insert Campaign + // Checking if there is enough remaining deposit + let remaining_for_channel = get_total_remaining_for_channel(&app.redis, &app.pool, &campaign).await?; + + if campaign.budget > remaining_for_channel { + return Err(ResponseError::Conflict("Not Enough budget for campaign".to_string())); + } + + // If the channel is being created, the amount spent is 0, therefore remaining = budget + update_remaining_for_campaign(&app.redis.clone(), campaign.id, campaign.budget).await?; - match insert_or_modify_campaign(&app.pool, &campaign, &app.redis).await { + // insert Campaign + match insert_campaign(&app.pool, &campaign).await { Err(error) => { - // error!(&app.logger, "{}", &error; "module" => "create_channel"); - return Err(ResponseError::Conflict("channel already exists".to_string())); + error!(&app.logger, "{}", &error; "module" => "create_campaign"); + return Err(ResponseError::Conflict("campaign already exists".to_string())); } Ok(false) => Err(error_response), _ => Ok(()), @@ -53,93 +63,144 @@ pub async fn create_campaign( Ok(success_response(serde_json::to_string(&campaign)?)) } +pub async fn update_campaign( + req: Request, + app: &Application, +) -> Result, ResponseError> { + let body = hyper::body::to_bytes(req.into_body()).await?; + + let campaign = serde_json::from_slice::(&body) + .map_err(|e| ResponseError::FailedValidation(e.to_string()))?; + + let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); + + // modify Campaign + + match modify_campaign(&app.pool, &campaign, &app.redis).await { + Err(error) => { + error!(&app.logger, "{:?}", &error; "module" => "update_campaign"); + return Err(ResponseError::Conflict("Error modifying campaign".to_string())); + } + Ok(false) => Err(error_response), + _ => Ok(()), + }?; + + let update_response = SuccessResponse { success: true }; + + Ok(success_response(serde_json::to_string(&campaign)?)) +} + // TODO: Double check redis calls -async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { +async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { let key = format!("adexCampaign:campaignSpent:{}", id); // campaignSpent tracks the portion of the budget which has already been spent let campaign_spent = match redis::cmd("GET") - .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) - .await? - { - Some(spent) => UnifiedNum::from(spent), - // TODO: Double check if this is true - // If the campaign is just being inserted, there would be no entry therefore no funds would be spent - None => UnifiedNum::from(0) - }; + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await + .map_err(|_| ResponseError::Conflict("Error getting campaignSpent for current campaign".to_string()))?{ + Some(spent) => { + // TODO: Fix unwraps + let res = BigNum::from_str(&spent).unwrap(); + let res = res.to_u64().unwrap(); + UnifiedNum::from_u64(res) + }, + None => UnifiedNum::from_u64(0) + }; + Ok(campaign_spent) } -async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { +async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { // update a key in Redis for the remaining spendable amount let key = format!("adexCampaign:remainingSpendable:{}", id); redis::cmd("SET") .arg(&key) .arg(amount.to_u64()) .query_async(&mut redis.clone()) - .await?; - Ok(true) + .await + .map_err(|_| ResponseError::Conflict("Error updating remainingSpendable for current campaign".to_string()))? } -async fn update_remaining_for_channel(redis: &MultiplexedConnection, id: ChannelId, amount: UnifiedNum) -> Result { - let key = format!("adexChannel:remaining:{}", id); - redis::cmd("SET") - .arg(&key) - .arg(amount.to_u64()) - .query_async(&mut redis.clone()) - .await?; - Ok(true) +async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign) -> Result { + // Getting the latest new state from Postgres + let latest_new_state = latest_new_state_v5(&pool, &campaign.channel, "").await?; + + let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + + let total_deposited = latest_spendable.deposit.total; + + let latest_new_state = latest_new_state.ok_or_else(|| ResponseError::Conflict("Error getting latest new state message".to_string()))?; + let msg = latest_new_state.msg; + let total_spent = msg.balances.get(&campaign.creator); + let zero = BigNum::from(0); + let total_spent = if let Some(spent) = total_spent { + spent + } else { + &zero + }; + + // TODO: total_spent is BigNum, is it safe to just convert it to UnifiedNum like this? + let total_spent = total_spent.to_u64().ok_or_else(|| { + ResponseError::Conflict("Error while converting total_spent to u64".to_string()) + }); + let total_remaining = total_deposited.checked_sub(&UnifiedNum::from_u64(total_spent)).ok_or_else(|| { + ResponseError::Conflict("Error while calculating the total remaining amount".to_string()) + })?; + Ok(total_remaining) } -async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign) -> Result { +// async fn update_remaining_for_channel(redis: &MultiplexedConnection, id: ChannelId, amount: UnifiedNum) -> Result { +// let key = format!("adexChannel:remaining:{}", id); +// redis::cmd("SET") +// .arg(&key) +// .arg(amount.to_u64()) +// .query_async(&mut redis.clone()) +// .await?; +// Ok(true) +// } + +async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign) -> Result { let campaigns_for_channel = get_campaigns_for_channel(&pool, &campaign).await?; let sum_of_campaigns_remaining = campaigns_for_channel .into_iter() - .map(async |c| { + .map(|c| async move { let spent = get_spent_for_campaign(&redis, c.id).await?; - let remaining = c.budget - spent; + let remaining = c.budget.checked_sub(&spent).ok_or_else(|| { + ResponseError::Conflict("Error while calculating remaining for campaign".to_string()) + }); remaining }) - .sum(); - Ok(sum_of_campaigns_remaining) + .fold(0u64, |sum: u64, val: u64| sum.checked_add(val).ok_or_else(|| { + return ResponseError::Conflict("Error while calculating total remaining for all campaigns".to_string()); + })); + Ok(UnifiedNum::from_u64(sum_of_campaigns_remaining)) } -pub async fn insert_or_modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { +pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { let campaign_spent = get_spent_for_campaign(&redis, campaign.id).await?; - // Check if we haven't exceeded the budget yet - if campaign.budget <= campaign_spent { + // Check if we have reached the budget + if campaign_spent >= campaign.budget { return Err(ResponseError::FailedValidation("No more budget available for spending".into())); } - let remaining_spendable_campaign = campaign.budget - campaign_spent; + let remaining_spendable_campaign = campaign.budget.checked_sub(&campaign_spent).ok_or_else(|| { + ResponseError::Conflict("Error while subtracting campaign_spent from budget".to_string()) + })?; update_remaining_for_campaign(&redis, campaign.id, remaining_spendable_campaign).await?; - // Getting the latest new state from Postgres - let latest_new_state = latest_new_state_v5(&pool, &campaign.channel, "").await?; - // Gets the latest Spendable for this (spender, channelId) pair - let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - let total_deposited = latest_spendable.deposit.total; - let total_spent = if let Some(lns) = latest_new_state { - lns.msg.into_inner().spenders[campaign.creator] - } else { - 0 - }; - - let total_remaining = total_deposited - total_spent; - - update_remaining_for_channel(&redis, campaign.channel.id(), total_remaining).await?; + // Gets the latest Spendable for this (spender, channelId) pair + let total_remaining = get_total_remaining_for_channel(&redis, &pool, &campaign).await?; - if campaign_exists(&pool, &campaign).await? { - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &pool, &campaign).await?; - if campaigns_remaining_sum > total_remaining { - return Err(ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel".into())); - } - return update_campaign(&pool, &campaign).await? + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &pool, &campaign).await?; + if campaigns_remaining_sum > total_remaining { + return Err(ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel".to_string())); } - return insert_campaign(&pool, &campaign).await?; + + update_campaign_in_db(&pool, &campaign).await.map_err(|e| ResponseError::Conflict(e.to_string())) // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] From f7dae5b60acc516d5b03ca86d869d5d8e03026aa Mon Sep 17 00:00:00 2001 From: simzzz Date: Thu, 10 Jun 2021 10:17:23 +0300 Subject: [PATCH 05/49] fixed typo --- sentry/src/db/campaign.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 62c2dd8fb..5f9c1211b 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -116,7 +116,7 @@ mod test { let exists = campaign_exists(&db_pool.clone(), campaign: &campaign_for_testing) .await .expect("Should succeed"); - asser!(exists); + assert!(exists); let fetched_campaign: Campaign = fetch_campaign(&db_pool, &campaign_for_testing) .await From 0f400160669c70b75b130bd7f7a7bbf5adf6eaf1 Mon Sep 17 00:00:00 2001 From: simzzz Date: Thu, 10 Jun 2021 16:42:46 +0300 Subject: [PATCH 06/49] Refactoring + changes to the way the remaining for all campaigns is received --- sentry/src/db/campaign.rs | 2 +- sentry/src/routes/campaign.rs | 76 ++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 5f9c1211b..43d29d507 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -113,7 +113,7 @@ mod test { assert!(is_inserted); - let exists = campaign_exists(&db_pool.clone(), campaign: &campaign_for_testing) + let exists = campaign_exists(&db_pool.clone(), &campaign_for_testing) .await .expect("Should succeed"); assert!(exists); diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 415e38aff..7a24dbfeb 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,5 +1,5 @@ use crate::{ - success_response, Application, Auth, ResponseError, RouteParams, Session, + success_response, Application, ResponseError, db::{ spendable::fetch_spendable, event_aggregate::latest_new_state_v5, @@ -11,16 +11,17 @@ use primitives::{ adapter::Adapter, sentry::{ campaign_create::CreateCampaign, - SuccessResponse + SuccessResponse, + MessageResponse }, - Campaign, CampaignId, UnifiedNum, ChannelId, BigNum + spender::Spendable, + validator::NewState, + Campaign, CampaignId, UnifiedNum, BigNum }; use redis::aio::MultiplexedConnection; -use deadpool_postgres::PoolError; use slog::error; -use tokio_postgres::error::SqlState; -use crate::db::campaign::{campaign_exists, update_campaign_in_db, insert_campaign, get_campaigns_for_channel, fetch_campaign}; -use std::{convert::TryFrom, fs, str::FromStr}; +use crate::db::campaign::{update_campaign_in_db, insert_campaign, get_campaigns_for_channel}; +use std::str::FromStr; pub async fn create_campaign( req: Request, @@ -39,7 +40,12 @@ pub async fn create_campaign( let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); // Checking if there is enough remaining deposit - let remaining_for_channel = get_total_remaining_for_channel(&app.redis, &app.pool, &campaign).await?; + // TODO: Switch with Accounting once it's ready + let latest_new_state = latest_new_state_v5(&app.pool, &campaign.channel, "").await?; + + // TODO: AIP#61: Update when changes to Spendable are ready + let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + let remaining_for_channel = get_total_remaining_for_channel(&app.redis, &app.pool, &campaign, &latest_new_state, &latest_spendable).await?; if campaign.budget > remaining_for_channel { return Err(ResponseError::Conflict("Not Enough budget for campaign".to_string())); @@ -122,16 +128,11 @@ async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: Campai .map_err(|_| ResponseError::Conflict("Error updating remainingSpendable for current campaign".to_string()))? } -async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign) -> Result { - // Getting the latest new state from Postgres - let latest_new_state = latest_new_state_v5(&pool, &campaign.channel, "").await?; - - let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - +async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign, latest_new_state: &Option>, latest_spendable: &Spendable) -> Result { let total_deposited = latest_spendable.deposit.total; - let latest_new_state = latest_new_state.ok_or_else(|| ResponseError::Conflict("Error getting latest new state message".to_string()))?; - let msg = latest_new_state.msg; + let latest_new_state = latest_new_state.as_ref().ok_or_else(|| ResponseError::Conflict("Error getting latest new state message".to_string()))?; + let msg = &latest_new_state.msg; let total_spent = msg.balances.get(&campaign.creator); let zero = BigNum::from(0); let total_spent = if let Some(spent) = total_spent { @@ -143,7 +144,7 @@ async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &D // TODO: total_spent is BigNum, is it safe to just convert it to UnifiedNum like this? let total_spent = total_spent.to_u64().ok_or_else(|| { ResponseError::Conflict("Error while converting total_spent to u64".to_string()) - }); + })?; let total_remaining = total_deposited.checked_sub(&UnifiedNum::from_u64(total_spent)).ok_or_else(|| { ResponseError::Conflict("Error while calculating the total remaining amount".to_string()) })?; @@ -160,26 +161,37 @@ async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &D // Ok(true) // } -async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign) -> Result { - let campaigns_for_channel = get_campaigns_for_channel(&pool, &campaign).await?; - let sum_of_campaigns_remaining = campaigns_for_channel +async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPool, campaigns: &Vec, mutated_campaign: &Campaign) -> Result { + let other_campaigns_remaining: Vec = campaigns .into_iter() + .filter(|c| c.id != mutated_campaign.id) .map(|c| async move { let spent = get_spent_for_campaign(&redis, c.id).await?; - let remaining = c.budget.checked_sub(&spent).ok_or_else(|| { - ResponseError::Conflict("Error while calculating remaining for campaign".to_string()) - }); - remaining + let remaining = c.budget.checked_sub(&spent)?; + Ok(remaining) }) - .fold(0u64, |sum: u64, val: u64| sum.checked_add(val).ok_or_else(|| { - return ResponseError::Conflict("Error while calculating total remaining for all campaigns".to_string()); - })); - Ok(UnifiedNum::from_u64(sum_of_campaigns_remaining)) + .collect(); + // TODO: Fix Unwrap + let sum_of_campaigns_remaining = other_campaigns_remaining + .into_iter() + .fold(UnifiedNum::from_u64(0), |mut sum, val| sum.checked_add(&val).unwrap()); + // Necessary to do it explicitly for current campaign as its budget is not yet updated in DB + let spent_for_mutated_campaign = get_spent_for_campaign(&redis, mutated_campaign.id).await?; + let remaining_for_mutated_campaign = mutated_campaign.budget.checked_sub(&spent_for_mutated_campaign).ok_or_else(|| { + ResponseError::Conflict("Error while calculating remaining for mutated campaign".to_string()) + })?; + sum_of_campaigns_remaining.checked_add(&remaining_for_mutated_campaign).ok_or_else(|| { + ResponseError::Conflict("Error while calculating sum for all campaigns".to_string()) + }); + Ok(sum_of_campaigns_remaining) } pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { let campaign_spent = get_spent_for_campaign(&redis, campaign.id).await?; + // Getting the latest new state from Postgres + let latest_new_state = latest_new_state_v5(&pool, &campaign.channel, "").await?; + let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; // Check if we have reached the budget if campaign_spent >= campaign.budget { return Err(ResponseError::FailedValidation("No more budget available for spending".into())); @@ -190,12 +202,10 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl })?; update_remaining_for_campaign(&redis, campaign.id, remaining_spendable_campaign).await?; - - // Gets the latest Spendable for this (spender, channelId) pair - let total_remaining = get_total_remaining_for_channel(&redis, &pool, &campaign).await?; - - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &pool, &campaign).await?; + let total_remaining = get_total_remaining_for_channel(&redis, &pool, &campaign, &latest_new_state, &latest_spendable).await?; + let campaigns_for_channel = get_campaigns_for_channel(&pool, &campaign).await?; + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &pool, &campaigns_for_channel, &campaign).await?; if campaigns_remaining_sum > total_remaining { return Err(ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel".to_string())); } From 33f0e3ea33bbaea9c0b6486d3b21aac61b25558b Mon Sep 17 00:00:00 2001 From: simzzz Date: Thu, 10 Jun 2021 16:58:30 +0300 Subject: [PATCH 07/49] added CampaignCreateResponse --- sentry/src/routes/campaign.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 7a24dbfeb..b99551d39 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -12,7 +12,7 @@ use primitives::{ sentry::{ campaign_create::CreateCampaign, SuccessResponse, - MessageResponse + MessageResponse, }, spender::Spendable, validator::NewState, @@ -27,6 +27,12 @@ pub async fn create_campaign( req: Request, app: &Application, ) -> Result, ResponseError> { + use serde::Serialize; + #[derive(Serialize)] + struct CreateCampaignResponse<'a> { + campaign: &'a Campaign, + } + let body = hyper::body::to_bytes(req.into_body()).await?; let campaign = serde_json::from_slice::(&body) @@ -64,9 +70,9 @@ pub async fn create_campaign( _ => Ok(()), }?; - let create_response = SuccessResponse { success: true }; + let create_response = CreateCampaignResponse { campaign: &campaign }; - Ok(success_response(serde_json::to_string(&campaign)?)) + Ok(success_response(serde_json::to_string(&create_response)?)) } pub async fn update_campaign( From 475b8db256ffe9047fd53d420f7301f6e959284b Mon Sep 17 00:00:00 2001 From: simzzz Date: Fri, 11 Jun 2021 16:42:12 +0300 Subject: [PATCH 08/49] More changes to budget/redis operations + fixed problems from PR review --- sentry/src/db/campaign.rs | 7 ++-- sentry/src/routes/campaign.rs | 71 +++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index fa49dcce5..eba32944f 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -49,9 +49,9 @@ pub async fn get_campaigns_for_channel(pool: &DbPool, campaign: &Campaign) -> Re let client = pool.get().await?; let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE channel_id = $1").await?; - let row = client.query(&statement, &[&campaign.channel.id()]).await?; + let rows = client.query(&statement, &[&campaign.channel.id()]).await?; - let campaigns = row.into_iter().map(|c| Campaign::from(&c)).collect(); + let campaigns = rows.iter().map(Campaign::from).collect(); Ok(campaigns) } @@ -67,7 +67,8 @@ pub async fn campaign_exists(pool: &DbPool, campaign: &Campaign) -> Result Result { +// TODO: Test for campaign ad_units +pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let statement = client .prepare("UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 WHERE id = $8") diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index b99551d39..647f89bae 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -20,19 +20,17 @@ use primitives::{ }; use redis::aio::MultiplexedConnection; use slog::error; -use crate::db::campaign::{update_campaign_in_db, insert_campaign, get_campaigns_for_channel}; -use std::str::FromStr; +use crate::db::campaign::{update_campaign as update_campaign_db, insert_campaign, get_campaigns_for_channel}; +use std::{ + cmp::max, + str::FromStr, +}; +use futures::future::join_all; pub async fn create_campaign( req: Request, app: &Application, ) -> Result, ResponseError> { - use serde::Serialize; - #[derive(Serialize)] - struct CreateCampaignResponse<'a> { - campaign: &'a Campaign, - } - let body = hyper::body::to_bytes(req.into_body()).await?; let campaign = serde_json::from_slice::(&body) @@ -70,9 +68,7 @@ pub async fn create_campaign( _ => Ok(()), }?; - let create_response = CreateCampaignResponse { campaign: &campaign }; - - Ok(success_response(serde_json::to_string(&create_response)?)) + Ok(success_response(serde_json::to_string(&campaign)?)) } pub async fn update_campaign( @@ -104,34 +100,51 @@ pub async fn update_campaign( // TODO: Double check redis calls async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { - let key = format!("adexCampaign:campaignSpent:{}", id); + let key = format!("spent:{}", id); // campaignSpent tracks the portion of the budget which has already been spent let campaign_spent = match redis::cmd("GET") .arg(&key) .query_async::<_, Option>(&mut redis.clone()) .await - .map_err(|_| ResponseError::Conflict("Error getting campaignSpent for current campaign".to_string()))?{ - Some(spent) => { - // TODO: Fix unwraps + { + Ok(Some(spent)) => { let res = BigNum::from_str(&spent).unwrap(); let res = res.to_u64().unwrap(); UnifiedNum::from_u64(res) }, - None => UnifiedNum::from_u64(0) + _ => UnifiedNum::from_u64(0) }; Ok(campaign_spent) } +async fn get_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { + let key = format!("remaining:{}", id); + let remaining = match redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await + { + Ok(Some(remaining)) => { + let res = BigNum::from_str(&remaining).unwrap(); + let res = res.to_u64().unwrap(); + UnifiedNum::from_u64(res) + }, + _ => UnifiedNum::from_u64(0) + }; + Ok(remaining) +} + async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { // update a key in Redis for the remaining spendable amount - let key = format!("adexCampaign:remainingSpendable:{}", id); - redis::cmd("SET") + let key = format!("remaining:{}", id); + redis::cmd("INCRBY") .arg(&key) .arg(amount.to_u64()) .query_async(&mut redis.clone()) .await - .map_err(|_| ResponseError::Conflict("Error updating remainingSpendable for current campaign".to_string()))? + .map_err(|_| ResponseError::Conflict("Error updating remainingSpendable for current campaign".to_string()))?; + Ok(true) } async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign, latest_new_state: &Option>, latest_spendable: &Spendable) -> Result { @@ -168,7 +181,7 @@ async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &D // } async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPool, campaigns: &Vec, mutated_campaign: &Campaign) -> Result { - let other_campaigns_remaining: Vec = campaigns + let other_campaigns_remaining = campaigns .into_iter() .filter(|c| c.id != mutated_campaign.id) .map(|c| async move { @@ -176,7 +189,8 @@ async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPoo let remaining = c.budget.checked_sub(&spent)?; Ok(remaining) }) - .collect(); + .collect::>(); + let other_campaigns_remaining = join_all(other_campaigns_remaining).await; // TODO: Fix Unwrap let sum_of_campaigns_remaining = other_campaigns_remaining .into_iter() @@ -203,10 +217,19 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl return Err(ResponseError::FailedValidation("No more budget available for spending".into())); } - let remaining_spendable_campaign = campaign.budget.checked_sub(&campaign_spent).ok_or_else(|| { + let old_spendable = get_remaining_for_campaign(&redis, campaign.id).await?; + + let new_spendable = campaign.budget.checked_sub(&campaign_spent).ok_or_else(|| { ResponseError::Conflict("Error while subtracting campaign_spent from budget".to_string()) })?; - update_remaining_for_campaign(&redis, campaign.id, remaining_spendable_campaign).await?; + + let diff_in_remaining = new_spendable.checked_sub(&old_spendable).ok_or_else(|| { + ResponseError::Conflict("Error while subtracting campaign_spent from budget".to_string()) + })?; + + let diff_in_remaining = max(UnifiedNum::from_u64(0), diff_in_remaining); + + update_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; // Gets the latest Spendable for this (spender, channelId) pair let total_remaining = get_total_remaining_for_channel(&redis, &pool, &campaign, &latest_new_state, &latest_spendable).await?; @@ -216,7 +239,7 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl return Err(ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel".to_string())); } - update_campaign_in_db(&pool, &campaign).await.map_err(|e| ResponseError::Conflict(e.to_string())) + update_campaign_db(&pool, &campaign).await.map_err(|e| ResponseError::Conflict(e.to_string())) // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] From 60efa4a6b3cfe2fb39d29c634c1dba353db5486c Mon Sep 17 00:00:00 2001 From: simzzz Date: Fri, 11 Jun 2021 17:27:01 +0300 Subject: [PATCH 09/49] more changes to updating remaining --- sentry/src/routes/campaign.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 647f89bae..4a10d9621 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -217,18 +217,17 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl return Err(ResponseError::FailedValidation("No more budget available for spending".into())); } - let old_spendable = get_remaining_for_campaign(&redis, campaign.id).await?; + let old_remaining = get_remaining_for_campaign(&redis, campaign.id).await?; + let old_remaining = UnifiedNum::from_u64(max(0, old_remaining.to_u64())); - let new_spendable = campaign.budget.checked_sub(&campaign_spent).ok_or_else(|| { + let new_remaining = campaign.budget.checked_sub(&campaign_spent).ok_or_else(|| { ResponseError::Conflict("Error while subtracting campaign_spent from budget".to_string()) })?; - let diff_in_remaining = new_spendable.checked_sub(&old_spendable).ok_or_else(|| { + let diff_in_remaining = new_remaining.checked_sub(&old_remaining).ok_or_else(|| { ResponseError::Conflict("Error while subtracting campaign_spent from budget".to_string()) })?; - let diff_in_remaining = max(UnifiedNum::from_u64(0), diff_in_remaining); - update_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; // Gets the latest Spendable for this (spender, channelId) pair From 688a51c682ded46913eb87f754e00363a6cf9cf9 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Sat, 12 Jun 2021 12:32:42 +0300 Subject: [PATCH 10/49] sentry - db - re-export campaign module --- sentry/src/db.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry/src/db.rs b/sentry/src/db.rs index 2f26d7b94..725aa1dcf 100644 --- a/sentry/src/db.rs +++ b/sentry/src/db.rs @@ -12,6 +12,7 @@ pub mod event_aggregate; pub mod spendable; mod validator_message; +pub use self::campaign::*; pub use self::channel::*; pub use self::event_aggregate::*; pub use self::validator_message::*; From 7b93256da9481c171fb72ea83ccc5173700c0559 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Sun, 13 Jun 2021 10:54:16 +0300 Subject: [PATCH 11/49] sentry - db - campaign - fix fetch_campaign --- sentry/src/db/campaign.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index b8753c5e0..42d0463c2 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -1,5 +1,5 @@ use crate::db::{DbPool, PoolError}; -use primitives::Campaign; +use primitives::{Campaign, CampaignId}; use tokio_postgres::types::Json; // TODO: Remove once we use this fn @@ -38,15 +38,13 @@ pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result Result { +pub async fn fetch_campaign(pool: DbPool, campaign: &CampaignId) -> Result, PoolError> { let client = pool.get().await?; let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE id = $1").await?; - let row = client.query_one(&statement, &[&campaign.id]).await?; + let row = client.query_opt(&statement, &[&campaign]).await?; - Ok(Campaign::from(&row)) + Ok(row.as_ref().map(Campaign::from)) } #[cfg(test)] @@ -66,16 +64,23 @@ mod test { .expect("Migrations should succeed"); let campaign_for_testing = DUMMY_CAMPAIGN.clone(); + + let non_existent_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing.id) + .await + .expect("Should fetch successfully"); + + assert_eq!(None, non_existent_campaign); + let is_inserted = insert_campaign(&database.pool, &campaign_for_testing) .await .expect("Should succeed"); assert!(is_inserted); - let fetched_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing) + let fetched_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing.id) .await .expect("Should fetch successfully"); - assert_eq!(campaign_for_testing, fetched_campaign); + assert_eq!(Some(campaign_for_testing), fetched_campaign); } } From f42d5b6aff19b7aea5a1701904a72a467cabfd75 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Sun, 13 Jun 2021 11:22:49 +0300 Subject: [PATCH 12/49] sentry - middleware - campaign - CampaignLoad --- sentry/src/db/campaign.rs | 11 ++- sentry/src/lib.rs | 6 +- sentry/src/middleware.rs | 1 + sentry/src/middleware/campaign.rs | 144 ++++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 sentry/src/middleware/campaign.rs diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 42d0463c2..240684572 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -38,7 +38,10 @@ pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result Result, PoolError> { +pub async fn fetch_campaign( + pool: DbPool, + campaign: &CampaignId, +) -> Result, PoolError> { let client = pool.get().await?; let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE id = $1").await?; @@ -64,13 +67,13 @@ mod test { .expect("Migrations should succeed"); let campaign_for_testing = DUMMY_CAMPAIGN.clone(); - + let non_existent_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing.id) .await .expect("Should fetch successfully"); - + assert_eq!(None, non_existent_campaign); - + let is_inserted = insert_campaign(&database.pool, &campaign_for_testing) .await .expect("Should succeed"); diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 403df405a..b727c93bb 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -60,8 +60,8 @@ lazy_static! { static ref CREATE_EVENTS_BY_CHANNEL_ID: Regex = Regex::new(r"^/channel/0x([a-zA-Z0-9]{64})/events/?$").expect("The regex should be valid"); } -#[derive(Debug)] -pub struct RouteParams(Vec); +#[derive(Debug, Clone)] +pub struct RouteParams(pub Vec); impl RouteParams { pub fn get(&self, index: usize) -> Option { @@ -308,7 +308,7 @@ async fn channels_router( } } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub enum ResponseError { NotFound, BadRequest(String), diff --git a/sentry/src/middleware.rs b/sentry/src/middleware.rs index f0a652d54..0b892f0b6 100644 --- a/sentry/src/middleware.rs +++ b/sentry/src/middleware.rs @@ -7,6 +7,7 @@ use primitives::adapter::Adapter; use async_trait::async_trait; pub mod auth; +pub mod campaign; pub mod channel; pub mod cors; diff --git a/sentry/src/middleware/campaign.rs b/sentry/src/middleware/campaign.rs new file mode 100644 index 000000000..42d16b281 --- /dev/null +++ b/sentry/src/middleware/campaign.rs @@ -0,0 +1,144 @@ +use crate::{db::fetch_campaign, middleware::Middleware}; +use crate::{Application, ResponseError, RouteParams}; +use hyper::{Body, Request}; +use primitives::adapter::Adapter; + +use async_trait::async_trait; + +#[derive(Debug)] +pub struct CampaignLoad; + +#[async_trait] +impl Middleware for CampaignLoad { + async fn call<'a>( + &self, + mut request: Request, + application: &'a Application, + ) -> Result, ResponseError> { + let id = request + .extensions() + .get::() + .ok_or_else(|| ResponseError::BadRequest("Route params not found".to_string()))? + .get(0) + .ok_or_else(|| ResponseError::BadRequest("No id".to_string()))?; + + let campaign_id = id + .parse() + .map_err(|_| ResponseError::BadRequest("Wrong Campaign Id".to_string()))?; + let campaign = fetch_campaign(application.pool.clone(), &campaign_id) + .await? + .ok_or(ResponseError::NotFound)?; + + request.extensions_mut().insert(campaign); + + Ok(request) + } +} + +#[cfg(test)] +mod test { + use adapter::DummyAdapter; + use primitives::{ + adapter::DummyAdapterOptions, + config::configuration, + util::tests::{ + discard_logger, + prep_db::{DUMMY_CAMPAIGN, IDS}, + }, + Campaign, + }; + + use crate::db::{ + insert_campaign, + redis_pool::TESTS_POOL, + tests_postgres::{setup_test_migrations, DATABASE_POOL}, + }; + + use super::*; + + async fn setup_app() -> Application { + let config = configuration("development", None).expect("Should get Config"); + let adapter = DummyAdapter::init( + DummyAdapterOptions { + dummy_identity: IDS["leader"], + dummy_auth: Default::default(), + dummy_auth_tokens: Default::default(), + }, + &config, + ); + + let redis = TESTS_POOL.get().await.expect("Should return Object"); + let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); + + setup_test_migrations(database.pool.clone()) + .await + .expect("Migrations should succeed"); + + let app = Application::new( + adapter, + config, + discard_logger(), + redis.connection.clone(), + database.pool.clone(), + ); + + app + } + + #[tokio::test] + async fn campaign_loading() { + let app = setup_app().await; + + let build_request = |params: RouteParams| { + Request::builder() + .extension(params) + .body(Body::empty()) + .expect("Should build Request") + }; + + let campaign = DUMMY_CAMPAIGN.clone(); + + let campaign_load = CampaignLoad; + + // bad CampaignId + { + let route_params = RouteParams(vec!["Bad campaign Id".to_string()]); + + let res = campaign_load + .call(build_request(route_params), &app) + .await + .expect_err("Should return error for Bad Campaign"); + + assert_eq!( + ResponseError::BadRequest("Wrong Campaign Id".to_string()), + res + ); + } + + let route_params = RouteParams(vec![campaign.id.to_string()]); + // non-existent campaign + { + let res = campaign_load + .call(build_request(route_params.clone()), &app) + .await + .expect_err("Should return error for Not Found"); + + assert!(matches!(res, ResponseError::NotFound)); + } + + // existing Campaign + { + // insert Campaign + assert!(insert_campaign(&app.pool, &campaign) + .await + .expect("Should insert Campaign")); + + let request = campaign_load + .call(build_request(route_params), &app) + .await + .expect("Should load campaign"); + + assert_eq!(Some(&campaign), request.extensions().get::()); + } + } +} From 1b8cd75edf0d9cbddf418cf4dc2d6541d37409c0 Mon Sep 17 00:00:00 2001 From: simzzz Date: Mon, 14 Jun 2021 16:10:53 +0300 Subject: [PATCH 13/49] added regex for campaign update route --- sentry/src/lib.rs | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 993e1eed4..b2dbd2634 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -60,6 +60,8 @@ lazy_static! { static ref ADVERTISER_ANALYTICS_BY_CHANNEL_ID: Regex = Regex::new(r"^/analytics/for-advertiser/0x([a-zA-Z0-9]{64})/?$").expect("The regex should be valid"); static ref PUBLISHER_ANALYTICS_BY_CHANNEL_ID: Regex = Regex::new(r"^/analytics/for-publisher/0x([a-zA-Z0-9]{64})/?$").expect("The regex should be valid"); static ref CREATE_EVENTS_BY_CHANNEL_ID: Regex = Regex::new(r"^/channel/0x([a-zA-Z0-9]{64})/events/?$").expect("The regex should be valid"); + static ref CAMPAIGN_UPDATE_BY_ID: Regex = + Regex::new(r"^/channel/0x([a-zA-Z0-9]{64})/?$").expect("The regex should be valid"); } #[derive(Debug)] @@ -163,21 +165,11 @@ impl Application { create_campaign(req, &self).await } - // For editing campaigns - ("/campaign/:id", &Method::POST) => { - let req = match AuthRequired.call(req, &self).await { - Ok(req) => req, - Err(error) => { - return map_response_error(error); - } - }; - - update_campaign(req, &self).await - } (route, _) if route.starts_with("/analytics") => analytics_router(req, &self).await, // This is important becuase it prevents us from doing // expensive regex matching for routes without /channel (path, _) if path.starts_with("/channel") => channels_router(req, &self).await, + (path, _) if path.starts_with("/campaign") => campaigns_router(req, &self).await, _ => Err(ResponseError::NotFound), } .unwrap_or_else(map_response_error); @@ -332,6 +324,28 @@ async fn channels_router( } } +async fn campaigns_router( + mut req: Request, + app: &Application, +) -> Result, ResponseError> { + req = AuthRequired.call(req, app).await?; + + let (path, method) = (req.uri().path().to_owned(), req.method()); + + // regex matching for routes with params + if let (Some(caps), &Method::POST) = (CAMPAIGN_UPDATE_BY_ID.captures(&path), method) { + let param = RouteParams(vec![caps + .get(1) + .map_or("".to_string(), |m| m.as_str().to_string())]); + + req.extensions_mut().insert(param); + + update_campaign(req, app).await + } else { + Err(ResponseError::NotFound) + } +} + #[derive(Debug)] pub enum ResponseError { NotFound, From be47fe2cac58795af122a743a1b77cbe9ce25c28 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 15 Jun 2021 14:43:49 +0300 Subject: [PATCH 14/49] sentry - route - campaign - insert events - Regex for matching Campaign Close & Campaign insert events --- sentry/src/lib.rs | 81 ++++++++++++++++++++++++++++++----- sentry/src/routes/campaign.rs | 49 ++++++++++++++++++++- 2 files changed, 117 insertions(+), 13 deletions(-) diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index b727c93bb..cb789481c 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -2,7 +2,7 @@ #![deny(rust_2018_idioms)] use crate::db::DbPool; -use crate::event_aggregator::EventAggregator; +use crate::routes::campaign; use crate::routes::channel::channel_status; use crate::routes::event_aggregate::list_channel_event_aggregates; use crate::routes::validator_message::{extract_params, list_validator_messages}; @@ -14,7 +14,8 @@ use middleware::{ channel::{ChannelLoad, GetChannelId}, cors::{cors, Cors}, }; -use middleware::{Chain, Middleware}; +use middleware::{campaign::CampaignLoad, Chain, Middleware}; +use once_cell::sync::Lazy; use primitives::adapter::Adapter; use primitives::sentry::ValidationErrorResponse; use primitives::{Config, ValidatorId}; @@ -23,8 +24,7 @@ use regex::Regex; use routes::analytics::{advanced_analytics, advertiser_analytics, analytics, publisher_analytics}; use routes::cfg::config; use routes::channel::{ - channel_list, channel_validate, create_channel, create_validator_messages, insert_events, - last_approved, + channel_list, channel_validate, create_channel, create_validator_messages, last_approved, }; use slog::Logger; use std::collections::HashMap; @@ -32,6 +32,7 @@ use std::collections::HashMap; pub mod middleware; pub mod routes { pub mod analytics; + pub mod campaign; pub mod cfg; pub mod channel; pub mod event_aggregate; @@ -41,8 +42,10 @@ pub mod routes { pub mod access; pub mod analytics_recorder; pub mod db; -pub mod event_aggregator; -pub mod event_reducer; +// TODO AIP#61: remove the even aggregator once we've taken out the logic for AIP#61 +// pub mod event_aggregator; +// TODO AIP#61: Remove even reducer or alter depending on our needs +// pub mod event_reducer; pub mod payout; pub mod spender; @@ -60,6 +63,13 @@ lazy_static! { static ref CREATE_EVENTS_BY_CHANNEL_ID: Regex = Regex::new(r"^/channel/0x([a-zA-Z0-9]{64})/events/?$").expect("The regex should be valid"); } +static INSERT_EVENTS_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { + Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/events/?$").expect("The regex should be valid") +}); +static CLOSE_CAMPAIGN_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { + Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/close/?$").expect("The regex should be valid") +}); + #[derive(Debug, Clone)] pub struct RouteParams(pub Vec); @@ -80,7 +90,6 @@ pub struct Application { pub redis: MultiplexedConnection, pub pool: DbPool, pub config: Config, - pub event_aggregator: EventAggregator, } impl Application { @@ -97,7 +106,6 @@ impl Application { logger, redis, pool, - event_aggregator: Default::default(), } } @@ -151,9 +159,10 @@ impl Application { publisher_analytics(req, &self).await } (route, _) if route.starts_with("/analytics") => analytics_router(req, &self).await, - // This is important becuase it prevents us from doing + // This is important because it prevents us from doing // expensive regex matching for routes without /channel (path, _) if path.starts_with("/channel") => channels_router(req, &self).await, + (path, _) if path.starts_with("/campaign") => campaigns_router(req, &self).await, _ => Err(ResponseError::NotFound), } .unwrap_or_else(map_response_error); @@ -164,12 +173,61 @@ impl Application { } } +async fn campaigns_router( + mut req: Request, + app: &Application, +) -> Result, ResponseError> { + let (path, method) = (req.uri().path(), req.method()); + + // create events + if let (Some(caps), &Method::POST) = (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) { + let param = RouteParams(vec![caps + .get(1) + .map_or("".to_string(), |m| m.as_str().to_string())]); + req.extensions_mut().insert(param); + + let req = CampaignLoad.call(req, app).await?; + + campaign::insert_events(req, app).await + } else if let (Some(_caps), &Method::POST) = + (CLOSE_CAMPAIGN_BY_CAMPAIGN_ID.captures(&path), method) + { + // TODO AIP#61: Close campaign: + // - only by creator + // - sets redis remaining = 0 (`newBudget = totalSpent`, i.e. `newBudget = oldBudget - remaining`) + + // let (is_creator, auth_uid) = match auth { + // Some(auth) => (auth.uid == channel.creator, auth.uid.to_string()), + // None => (false, Default::default()), + // }; + // Closing a campaign is allowed only by the creator + // if has_close_event && is_creator { + // return Ok(()); + // } + + Err(ResponseError::NotFound) + } else { + Err(ResponseError::NotFound) + } +} + async fn analytics_router( mut req: Request, app: &Application, ) -> Result, ResponseError> { let (route, method) = (req.uri().path(), req.method()); + + + // TODO AIP#61: Add routes for: + // - POST /channel/:id/pay + // #[serde(rename_all = "camelCase")] + // Pay { payout: BalancesMap }, + // + // - GET /channel/:id/spender/:addr + // - GET /channel/:id/spender/all + // - POST /channel/:id/spender/:addr + // - GET /channel/:id/get-leaf match *method { Method::GET => { if let Some(caps) = ANALYTICS_BY_CHANNEL_ID.captures(route) { @@ -229,7 +287,7 @@ async fn channels_router( let (path, method) = (req.uri().path().to_owned(), req.method()); // regex matching for routes with params - if let (Some(caps), &Method::POST) = (CREATE_EVENTS_BY_CHANNEL_ID.captures(&path), method) { + /* if let (Some(caps), &Method::POST) = (CREATE_EVENTS_BY_CHANNEL_ID.captures(&path), method) { let param = RouteParams(vec![caps .get(1) .map_or("".to_string(), |m| m.as_str().to_string())]); @@ -237,7 +295,8 @@ async fn channels_router( req.extensions_mut().insert(param); insert_events(req, app).await - } else if let (Some(caps), &Method::GET) = (LAST_APPROVED_BY_CHANNEL_ID.captures(&path), method) + } else */ + if let (Some(caps), &Method::GET) = (LAST_APPROVED_BY_CHANNEL_ID.captures(&path), method) { let param = RouteParams(vec![caps .get(1) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 94076e34b..e20850f9c 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,7 +1,8 @@ +use std::collections::HashMap; + use crate::{success_response, Application, Auth, ResponseError, RouteParams, Session}; use hyper::{Body, Request, Response}; -use primitives::{adapter::Adapter, sentry::{ - campaign_create::CreateCampaign,SuccessResponse}}; +use primitives::{CampaignId, adapter::Adapter, sentry::{Event, SuccessResponse, campaign_create::CreateCampaign}}; pub async fn create_campaign( req: Request, @@ -42,3 +43,47 @@ pub async fn create_campaign( Ok(success_response(serde_json::to_string(&campaign)?)) } + + +pub async fn insert_events( + req: Request, + app: &Application, +) -> Result, ResponseError> { + let (req_head, req_body) = req.into_parts(); + + let auth = req_head.extensions.get::(); + let session = req_head + .extensions + .get::() + .expect("request should have session"); + + let route_params = req_head + .extensions + .get::() + .expect("request should have route params"); + + let campaign_id: CampaignId = route_params.index(0).parse()?; + + let body_bytes = hyper::body::to_bytes(req_body).await?; + let mut request_body = serde_json::from_slice::>>(&body_bytes)?; + + let events = request_body + .remove("events") + .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; + + // + // TODO #381: AIP#61 Spender Aggregator should be called + // + + // handle events - check access + // handle events - Update targeting rules + // calculate payout + // distribute fees + // handle spending - Spender Aggregate + // handle events - aggregate Events and put into analytics + + Ok(Response::builder() + .header("Content-type", "application/json") + .body(serde_json::to_string(&SuccessResponse { success: true })?.into()) + .unwrap()) +} \ No newline at end of file From 00abea5964b30583e8aed9b34635bbbe6be48421 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 15 Jun 2021 14:49:31 +0300 Subject: [PATCH 15/49] primitives - sentry - remove Events: - removed Update targeting, Close & Pay events - sentry - comment out channel::Insert_events - sentry - access - remove unnecessary checks & use Campaign instead of Channel --- primitives/src/sentry.rs | 16 - sentry/src/access.rs | 622 ++++++++++++++------------------- sentry/src/db/channel.rs | 1 + sentry/src/event_aggregator.rs | 61 ++-- sentry/src/payout.rs | 30 -- sentry/src/routes/campaign.rs | 66 +++- sentry/src/routes/channel.rs | 92 ++--- 7 files changed, 400 insertions(+), 488 deletions(-) diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index da6bbacf2..afd8a6214 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -125,19 +125,6 @@ pub enum Event { ad_slot: Option, referrer: Option, }, - /// only the creator can send this event - #[serde(rename_all = "camelCase")] - UpdateTargeting { targeting_rules: Rules }, - /// Closes the `Campaign` - /// only the creator can send this event - #[serde(rename_all = "camelCase")] - Close, - /// TODO: AIP#61 Check and explain who can send this event as well as when it can be received - /// A map of earners which gets merged in the `spender::Aggregate` - /// NOTE: Does **not** contain any fees! - /// This even can be used to pay to yourself, but this is irrelevant as it's your funds you are paying yourself. - #[serde(rename_all = "camelCase")] - Pay { payout: BalancesMap }, } impl Event { @@ -159,9 +146,6 @@ impl AsRef for Event { match *self { Event::Impression { .. } => "IMPRESSION", Event::Click { .. } => "CLICK", - Event::UpdateTargeting { .. } => "UPDATE_TARGETING", - Event::Close => "CLOSE", - Event::Pay { .. } => "PAY", } } } diff --git a/sentry/src/access.rs b/sentry/src/access.rs index 2802fdb71..cc49caf7c 100644 --- a/sentry/src/access.rs +++ b/sentry/src/access.rs @@ -3,22 +3,18 @@ use futures::future::try_join_all; use redis::aio::MultiplexedConnection; use crate::{Auth, Session}; -use primitives::event_submission::{RateLimit, Rule}; -use primitives::sentry::Event; -use primitives::Channel; +use primitives::{ + event_submission::{RateLimit, Rule}, + sentry::Event, + Campaign, +}; use std::cmp::PartialEq; use thiserror::Error; #[derive(Debug, PartialEq, Eq, Error)] pub enum Error { - #[error("only creator can close channel")] - OnlyCreatorCanCloseChannel, - #[error("only creator can update targeting rules")] - OnlyCreatorCanUpdateTargetingRules, #[error("channel is expired")] - ChannelIsExpired, - #[error("channel is in withdraw period")] - ChannelIsInWithdrawPeriod, + CampaignIsExpired, #[error("event submission restricted")] ForbiddenReferrer, #[error("{0}")] @@ -33,61 +29,31 @@ pub async fn check_access( session: &Session, auth: Option<&Auth>, rate_limit: &RateLimit, - channel: &Channel, + campaign: &Campaign, events: &[Event], ) -> Result<(), Error> { - let is_close_event = |e: &Event| matches!(e, Event::Close); - let is_update_targeting_event = |e: &Event| matches!(e, Event::UpdateTargeting { .. }); - - let has_close_event = events.iter().all(is_close_event); - let has_update_targeting_event = events.iter().all(is_update_targeting_event); let current_time = Utc::now(); - let is_in_withdraw_period = current_time > channel.spec.withdraw_period_start; - - if current_time > channel.valid_until { - return Err(Error::ChannelIsExpired); - } - if has_close_event && is_in_withdraw_period { - return Ok(()); + if current_time > campaign.active.to { + return Err(Error::CampaignIsExpired); } let (is_creator, auth_uid) = match auth { - Some(auth) => (auth.uid == channel.creator, auth.uid.to_string()), + Some(auth) => ( + auth.uid.to_address() == campaign.creator, + auth.uid.to_string(), + ), None => (false, Default::default()), }; - // We're only sending a CLOSE - // That's allowed for the creator normally, and for everyone during the withdraw period - if has_close_event && is_creator { - return Ok(()); - } - - if has_update_targeting_event && is_creator { - return Ok(()); - } - - // Only the creator can send a CLOSE - if !is_creator && events.iter().any(is_close_event) { - return Err(Error::OnlyCreatorCanCloseChannel); - } - - // Only the creator can send a UPDATE_TARGETING - if !is_creator && events.iter().any(is_update_targeting_event) { - return Err(Error::OnlyCreatorCanUpdateTargetingRules); - } - - if is_in_withdraw_period { - return Err(Error::ChannelIsInWithdrawPeriod); - } - // Extra rulfes for normal (non-CLOSE) events + // Rules for events if forbidden_country(&session) || forbidden_referrer(&session) { return Err(Error::ForbiddenReferrer); } let default_rules = [ Rule { - uids: Some(vec![channel.creator.to_string()]), + uids: Some(vec![campaign.creator.to_string()]), rate_limit: None, }, Rule { @@ -97,8 +63,7 @@ pub async fn check_access( ]; // Enforce access limits - let allow_rules = channel - .spec + let allow_rules = campaign .event_submission .as_ref() .map(|ev_sub| ev_sub.allow.as_slice()) @@ -118,11 +83,16 @@ pub async fn check_access( return Ok(()); } - let apply_all_rules = try_join_all( - rules - .iter() - .map(|rule| apply_rule(redis.clone(), &rule, &events, &channel, &auth_uid, &session)), - ); + let apply_all_rules = try_join_all(rules.iter().map(|rule| { + apply_rule( + redis.clone(), + &rule, + &events, + &campaign, + &auth_uid, + &session, + ) + })); if let Err(rule_error) = apply_all_rules.await { Err(Error::RulesError(rule_error)) @@ -135,21 +105,25 @@ async fn apply_rule( redis: MultiplexedConnection, rule: &Rule, events: &[Event], - channel: &Channel, + campaign: &Campaign, uid: &str, session: &Session, ) -> Result<(), String> { match &rule.rate_limit { Some(rate_limit) => { let key = if &rate_limit.limit_type == "sid" { - Ok(format!("adexRateLimit:{}:{}", hex::encode(channel.id), uid)) + Ok(format!( + "adexRateLimit:{}:{}", + hex::encode(campaign.id), + uid + )) } else if &rate_limit.limit_type == "ip" { if events.len() != 1 { Err("rateLimit: only allows 1 event".to_string()) } else { Ok(format!( "adexRateLimit:{}:{}", - hex::encode(channel.id), + hex::encode(campaign.id), session.ip.as_ref().unwrap_or(&String::new()) )) } @@ -215,7 +189,7 @@ mod test { event_submission::{RateLimit, Rule}, sentry::Event, targeting::Rules, - util::tests::prep_db::{ADDRESSES, DUMMY_CHANNEL, IDS}, + util::tests::prep_db::{ADDRESSES, DUMMY_CAMPAIGN, DUMMY_CHANNEL, IDS}, Channel, Config, EventSubmission, }; @@ -235,14 +209,14 @@ mod test { (config, connection) } - fn get_channel(with_rule: Rule) -> Channel { - let mut channel = DUMMY_CHANNEL.clone(); + fn get_campaign(with_rule: Rule) -> Campaign { + let mut campaign = DUMMY_CAMPAIGN.clone(); - channel.spec.event_submission = Some(EventSubmission { + campaign.event_submission = Some(EventSubmission { allow: vec![with_rule], }); - channel + campaign } fn get_impression_events(count: i8) -> Vec { @@ -256,18 +230,6 @@ mod test { .collect() } - fn get_close_events(count: i8) -> Vec { - (0..count).map(|_| Event::Close).collect() - } - - fn get_update_targeting_events(count: i8) -> Vec { - (0..count) - .map(|_| Event::UpdateTargeting { - targeting_rules: Rules::new(), - }) - .collect() - } - #[tokio::test] async fn session_uid_rate_limit() { let (config, database) = setup().await; @@ -292,14 +254,14 @@ mod test { }), }; let events = get_impression_events(2); - let channel = get_channel(rule); + let campaign = get_campaign(rule); let response = check_access( &database, &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &events, ) .await; @@ -310,7 +272,7 @@ mod test { &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &events, ) .await; @@ -345,14 +307,14 @@ mod test { time_frame: Duration::from_millis(1), }), }; - let channel = get_channel(rule); + let campaign = get_campaign(rule); let err_response = check_access( &database, &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &get_impression_events(2), ) .await; @@ -369,7 +331,7 @@ mod test { &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &get_impression_events(1), ) .await; @@ -399,276 +361,237 @@ mod test { time_frame: Duration::from_millis(1), }), }; - let mut channel = get_channel(rule); - channel.valid_until = Utc.ymd(1970, 1, 1).and_hms(12, 00, 9); + let mut campaign = get_campaign(rule); + campaign.active.to = Utc.ymd(1970, 1, 1).and_hms(12, 00, 9); let err_response = check_access( &database, &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &get_impression_events(2), ) .await; - assert_eq!(Err(Error::ChannelIsExpired), err_response); - } - - #[tokio::test] - async fn check_access_close_event_in_withdraw_period() { - let (config, database) = setup().await; - - let auth = Auth { - era: 0, - uid: IDS["follower"], - }; - - let session = Session { - ip: Default::default(), - referrer_header: None, - country: None, - os: None, - }; - - let rule = Rule { - uids: None, - rate_limit: Some(RateLimit { - limit_type: "ip".to_string(), - time_frame: Duration::from_millis(1), - }), - }; - let mut channel = get_channel(rule); - channel.spec.withdraw_period_start = Utc.ymd(1970, 1, 1).and_hms(12, 0, 9); - - let ok_response = check_access( - &database, - &session, - Some(&auth), - &config.ip_rate_limit, - &channel, - &get_close_events(1), - ) - .await; - - assert_eq!(Ok(()), ok_response); - } - - #[tokio::test] - async fn check_access_close_event_and_is_creator() { - let (config, database) = setup().await; - - let auth = Auth { - era: 0, - uid: IDS["follower"], - }; - - let session = Session { - ip: Default::default(), - referrer_header: None, - country: None, - os: None, - }; - - let rule = Rule { - uids: None, - rate_limit: Some(RateLimit { - limit_type: "ip".to_string(), - time_frame: Duration::from_millis(1), - }), - }; - let mut channel = get_channel(rule); - channel.creator = IDS["follower"]; - - let ok_response = check_access( - &database, - &session, - Some(&auth), - &config.ip_rate_limit, - &channel, - &get_close_events(1), - ) - .await; - - assert_eq!(Ok(()), ok_response); - } - - #[tokio::test] - async fn check_access_update_targeting_event_and_is_creator() { - let (config, database) = setup().await; - - let auth = Auth { - era: 0, - uid: IDS["follower"], - }; - - let session = Session { - ip: Default::default(), - referrer_header: None, - country: None, - os: None, - }; - - let rule = Rule { - uids: None, - rate_limit: Some(RateLimit { - limit_type: "ip".to_string(), - time_frame: Duration::from_millis(1), - }), - }; - let mut channel = get_channel(rule); - channel.creator = IDS["follower"]; - - let ok_response = check_access( - &database, - &session, - Some(&auth), - &config.ip_rate_limit, - &channel, - &get_update_targeting_events(1), - ) - .await; - - assert_eq!(Ok(()), ok_response); - } - - #[tokio::test] - async fn not_creator_and_there_are_close_events() { - let (config, database) = setup().await; - - let auth = Auth { - era: 0, - uid: IDS["follower"], - }; - - let session = Session { - ip: Default::default(), - referrer_header: None, - country: None, - os: None, - }; - - let rule = Rule { - uids: None, - rate_limit: Some(RateLimit { - limit_type: "ip".to_string(), - time_frame: Duration::from_millis(1), - }), - }; - let mut channel = get_channel(rule); - channel.creator = IDS["leader"]; - let mixed_events = vec![ - Event::Impression { - publisher: ADDRESSES["publisher2"], - ad_unit: None, - ad_slot: None, - referrer: None, - }, - Event::Close, - Event::UpdateTargeting { - targeting_rules: Rules::new(), - }, - ]; - let err_response = check_access( - &database, - &session, - Some(&auth), - &config.ip_rate_limit, - &channel, - &mixed_events, - ) - .await; - - assert_eq!(Err(Error::OnlyCreatorCanCloseChannel), err_response); - } - - #[tokio::test] - async fn not_creator_and_there_are_update_targeting_events() { - let (config, database) = setup().await; - - let auth = Auth { - era: 0, - uid: IDS["follower"], - }; - - let session = Session { - ip: Default::default(), - referrer_header: None, - country: None, - os: None, - }; - - let rule = Rule { - uids: None, - rate_limit: Some(RateLimit { - limit_type: "ip".to_string(), - time_frame: Duration::from_millis(1), - }), - }; - let mut channel = get_channel(rule); - channel.creator = IDS["leader"]; - let mixed_events = vec![ - Event::Impression { - publisher: ADDRESSES["publisher2"], - ad_unit: None, - ad_slot: None, - referrer: None, - }, - Event::UpdateTargeting { - targeting_rules: Rules::new(), - }, - ]; - let err_response = check_access( - &database, - &session, - Some(&auth), - &config.ip_rate_limit, - &channel, - &mixed_events, - ) - .await; - - assert_eq!(Err(Error::OnlyCreatorCanUpdateTargetingRules), err_response); + assert_eq!(Err(Error::CampaignIsExpired), err_response); } - #[tokio::test] - async fn in_withdraw_period_no_close_events() { - let (config, database) = setup().await; - - let auth = Auth { - era: 0, - uid: IDS["follower"], - }; - - let session = Session { - ip: Default::default(), - referrer_header: None, - country: None, - os: None, - }; - - let rule = Rule { - uids: None, - rate_limit: Some(RateLimit { - limit_type: "ip".to_string(), - time_frame: Duration::from_millis(1), - }), - }; - let mut channel = get_channel(rule); - channel.spec.withdraw_period_start = Utc.ymd(1970, 1, 1).and_hms(12, 0, 9); - - let err_response = check_access( - &database, - &session, - Some(&auth), - &config.ip_rate_limit, - &channel, - &get_impression_events(2), - ) - .await; - - assert_eq!(Err(Error::ChannelIsInWithdrawPeriod), err_response); - } + // #[tokio::test] + // async fn check_access_close_event_in_withdraw_period() { + // let (config, database) = setup().await; + + // let auth = Auth { + // era: 0, + // uid: IDS["follower"], + // }; + + // let session = Session { + // ip: Default::default(), + // referrer_header: None, + // country: None, + // os: None, + // }; + + // let rule = Rule { + // uids: None, + // rate_limit: Some(RateLimit { + // limit_type: "ip".to_string(), + // time_frame: Duration::from_millis(1), + // }), + // }; + // let mut channel = get_channel(rule); + // channel.spec.withdraw_period_start = Utc.ymd(1970, 1, 1).and_hms(12, 0, 9); + + // let ok_response = check_access( + // &database, + // &session, + // Some(&auth), + // &config.ip_rate_limit, + // &channel, + // &get_close_events(1), + // ) + // .await; + + // assert_eq!(Ok(()), ok_response); + // } + + // #[tokio::test] + // async fn check_access_close_event_and_is_creator() { + // let (config, database) = setup().await; + + // let auth = Auth { + // era: 0, + // uid: IDS["follower"], + // }; + + // let session = Session { + // ip: Default::default(), + // referrer_header: None, + // country: None, + // os: None, + // }; + + // let rule = Rule { + // uids: None, + // rate_limit: Some(RateLimit { + // limit_type: "ip".to_string(), + // time_frame: Duration::from_millis(1), + // }), + // }; + // let mut channel = get_channel(rule); + // channel.creator = IDS["follower"]; + + // let ok_response = check_access( + // &database, + // &session, + // Some(&auth), + // &config.ip_rate_limit, + // &channel, + // &get_close_events(1), + // ) + // .await; + + // assert_eq!(Ok(()), ok_response); + // } + + // #[tokio::test] + // async fn check_access_update_targeting_event_and_is_creator() { + // let (config, database) = setup().await; + + // let auth = Auth { + // era: 0, + // uid: IDS["follower"], + // }; + + // let session = Session { + // ip: Default::default(), + // referrer_header: None, + // country: None, + // os: None, + // }; + + // let rule = Rule { + // uids: None, + // rate_limit: Some(RateLimit { + // limit_type: "ip".to_string(), + // time_frame: Duration::from_millis(1), + // }), + // }; + // let mut channel = get_channel(rule); + // channel.creator = IDS["follower"]; + + // let ok_response = check_access( + // &database, + // &session, + // Some(&auth), + // &config.ip_rate_limit, + // &channel, + // &get_update_targeting_events(1), + // ) + // .await; + + // assert_eq!(Ok(()), ok_response); + // } + + // #[tokio::test] + // async fn not_creator_and_there_are_close_events() { + // let (config, database) = setup().await; + + // let auth = Auth { + // era: 0, + // uid: IDS["follower"], + // }; + + // let session = Session { + // ip: Default::default(), + // referrer_header: None, + // country: None, + // os: None, + // }; + + // let rule = Rule { + // uids: None, + // rate_limit: Some(RateLimit { + // limit_type: "ip".to_string(), + // time_frame: Duration::from_millis(1), + // }), + // }; + // let mut channel = get_channel(rule); + // channel.creator = IDS["leader"]; + // let mixed_events = vec![ + // Event::Impression { + // publisher: ADDRESSES["publisher2"], + // ad_unit: None, + // ad_slot: None, + // referrer: None, + // }, + // Event::Close, + // Event::UpdateTargeting { + // targeting_rules: Rules::new(), + // }, + // ]; + // let err_response = check_access( + // &database, + // &session, + // Some(&auth), + // &config.ip_rate_limit, + // &channel, + // &mixed_events, + // ) + // .await; + + // assert_eq!(Err(Error::OnlyCreatorCanCloseChannel), err_response); + // } + + // #[tokio::test] + // async fn not_creator_and_there_are_update_targeting_events() { + // let (config, database) = setup().await; + + // let auth = Auth { + // era: 0, + // uid: IDS["follower"], + // }; + + // let session = Session { + // ip: Default::default(), + // referrer_header: None, + // country: None, + // os: None, + // }; + + // let rule = Rule { + // uids: None, + // rate_limit: Some(RateLimit { + // limit_type: "ip".to_string(), + // time_frame: Duration::from_millis(1), + // }), + // }; + // let mut channel = get_channel(rule); + // channel.creator = IDS["leader"]; + // let mixed_events = vec![ + // Event::Impression { + // publisher: ADDRESSES["publisher2"], + // ad_unit: None, + // ad_slot: None, + // referrer: None, + // }, + // Event::UpdateTargeting { + // targeting_rules: Rules::new(), + // }, + // ]; + // let err_response = check_access( + // &database, + // &session, + // Some(&auth), + // &config.ip_rate_limit, + // &channel, + // &mixed_events, + // ) + // .await; + + // assert_eq!(Err(Error::OnlyCreatorCanUpdateTargetingRules), err_response); + // } #[tokio::test] async fn with_forbidden_country() { @@ -693,14 +616,14 @@ mod test { time_frame: Duration::from_millis(1), }), }; - let channel = get_channel(rule); + let campaign = get_campaign(rule); let err_response = check_access( &database, &session, Some(&auth), &config.ip_rate_limit, - &channel, + &&campaign, &get_impression_events(2), ) .await; @@ -731,14 +654,14 @@ mod test { time_frame: Duration::from_millis(1), }), }; - let channel = get_channel(rule); + let campaign = get_campaign(rule); let err_response = check_access( &database, &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &get_impression_events(2), ) .await; @@ -766,14 +689,14 @@ mod test { uids: None, rate_limit: None, }; - let channel = get_channel(rule); + let campaign = get_campaign(rule); let ok_response = check_access( &database, &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &get_impression_events(1), ) .await; @@ -804,21 +727,20 @@ mod test { time_frame: Duration::from_millis(60_000), }), }; - let channel = get_channel(rule); + let campaign = get_campaign(rule); let ok_response = check_access( &database, &session, Some(&auth), &config.ip_rate_limit, - &channel, + &campaign, &get_impression_events(1), ) .await; assert_eq!(Ok(()), ok_response); - let key = "adexRateLimit:061d5e2a67d0a9a10f1c732bca12a676d83f79663a396f7d87b3e30b9b411088:" - .to_string(); + let key = "adexRateLimit:936da01f9abd4d9d80c702af85c822a8:".to_string(); let value = "1".to_string(); let value_in_redis = redis::cmd("GET") diff --git a/sentry/src/db/channel.rs b/sentry/src/db/channel.rs index 6ec72eadf..528cc2dde 100644 --- a/sentry/src/db/channel.rs +++ b/sentry/src/db/channel.rs @@ -61,6 +61,7 @@ pub async fn insert_channel(pool: &DbPool, channel: &Channel) -> Result { - ResponseError::Forbidden(e.to_string()) - } - AccessError::OnlyCreatorCanUpdateTargetingRules => { - ResponseError::Forbidden(e.to_string()) - } - AccessError::RulesError(error) => ResponseError::TooManyRequests(error), - AccessError::UnAuthenticated => ResponseError::Unauthorized, - _ => ResponseError::BadRequest(e.to_string()), - })?; - - let new_targeting_rules = events.iter().find_map(|ev| match ev { - Event::UpdateTargeting { targeting_rules } => Some(targeting_rules), - _ => None, - }); - - if let Some(new_rules) = new_targeting_rules { - update_targeting_rules(&dbpool.clone(), &channel_id, &new_rules).await?; - } + // check_access( + // &app.redis, + // session, + // auth, + // &app.config.ip_rate_limit, + // &record.channel, + // &events, + // ) + // .await + // .map_err(|e| match e { + // AccessError::OnlyCreatorCanCloseChannel | AccessError::ForbiddenReferrer => { + // ResponseError::Forbidden(e.to_string()) + // } + // AccessError::OnlyCreatorCanUpdateTargetingRules => { + // ResponseError::Forbidden(e.to_string()) + // } + // AccessError::RulesError(error) => ResponseError::TooManyRequests(error), + // AccessError::UnAuthenticated => ResponseError::Unauthorized, + // _ => ResponseError::BadRequest(e.to_string()), + // })?; + + // let new_targeting_rules = events.iter().find_map(|ev| match ev { + // Event::UpdateTargeting { targeting_rules } => Some(targeting_rules), + // _ => None, + // }); + + // if let Some(new_rules) = new_targeting_rules { + // update_targeting_rules(&dbpool.clone(), &channel_id, &new_rules).await?; + // } // // TODO: AIP#61 Events & payouts should be separated in to Analytics & Spender Aggregator diff --git a/sentry/src/payout.rs b/sentry/src/payout.rs index ccc364467..e0ad94930 100644 --- a/sentry/src/payout.rs +++ b/sentry/src/payout.rs @@ -175,34 +175,4 @@ mod test { let expected_option = Some((ADDRESSES["leader"], 23.into())); assert_eq!(expected_option, payout, "pricingBounds: click event"); } - - #[test] - fn get_event_payouts_pricing_bounds_close_event() { - let logger = discard_logger(); - let mut campaign = DUMMY_CAMPAIGN.clone(); - campaign.budget = 100.into(); - campaign.pricing_bounds = Some(PricingBounds { - impression: Some(Pricing { - min: 8.into(), - max: 64.into(), - }), - click: Some(Pricing { - min: 23.into(), - max: 100.into(), - }), - }); - - let event = Event::Close; - - let session = Session { - ip: None, - country: None, - referrer_header: None, - os: None, - }; - - let payout = get_payout(&logger, &campaign, &event, &session).expect("Should be OK"); - - assert_eq!(None, payout, "pricingBounds: click event"); - } } diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index e20850f9c..5936b8974 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,8 +1,16 @@ use std::collections::HashMap; -use crate::{success_response, Application, Auth, ResponseError, RouteParams, Session}; +use crate::{ + access::{self, check_access}, + success_response, Application, Auth, ResponseError, Session, +}; +use chrono::Utc; use hyper::{Body, Request, Response}; -use primitives::{CampaignId, adapter::Adapter, sentry::{Event, SuccessResponse, campaign_create::CreateCampaign}}; +use primitives::{ + adapter::Adapter, + sentry::{campaign_create::CreateCampaign, Event, SuccessResponse}, + Campaign, +}; pub async fn create_campaign( req: Request, @@ -15,13 +23,12 @@ pub async fn create_campaign( // create the actual `Campaign` with random `CampaignId` .into_campaign(); - // TODO AIP#61: Validate Campaign let error_response = ResponseError::BadRequest("err occurred; please try again later".into()); // insert Campaign - + // match insert_campaign(&app.pool, &campaign).await { // Err(error) => { // error!(&app.logger, "{}", &error; "module" => "create_channel"); @@ -44,7 +51,6 @@ pub async fn create_campaign( Ok(success_response(serde_json::to_string(&campaign)?)) } - pub async fn insert_events( req: Request, app: &Application, @@ -57,12 +63,10 @@ pub async fn insert_events( .get::() .expect("request should have session"); - let route_params = req_head + let campaign = req_head .extensions - .get::() - .expect("request should have route params"); - - let campaign_id: CampaignId = route_params.index(0).parse()?; + .get::() + .expect("request should have a Campaign loaded"); let body_bytes = hyper::body::to_bytes(req_body).await?; let mut request_body = serde_json::from_slice::>>(&body_bytes)?; @@ -71,6 +75,25 @@ pub async fn insert_events( .remove("events") .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; + let processed = process_events(app, auth, session, campaign, events).await?; + + Ok(Response::builder() + .header("Content-type", "application/json") + .body(serde_json::to_string(&SuccessResponse { success: processed })?.into()) + .unwrap()) +} + +async fn process_events( + app: &Application, + auth: Option<&Auth>, + session: &Session, + campaign: &Campaign, + events: Vec, +) -> Result { + if &Utc::now() > &campaign.active.to { + return Err(ResponseError::BadRequest("Campaign is expired".into())); + } + // // TODO #381: AIP#61 Spender Aggregator should be called // @@ -82,8 +105,21 @@ pub async fn insert_events( // handle spending - Spender Aggregate // handle events - aggregate Events and put into analytics - Ok(Response::builder() - .header("Content-type", "application/json") - .body(serde_json::to_string(&SuccessResponse { success: true })?.into()) - .unwrap()) -} \ No newline at end of file + check_access( + &app.redis, + session, + auth, + &app.config.ip_rate_limit, + &campaign, + &events, + ) + .await + .map_err(|e| match e { + access::Error::ForbiddenReferrer => ResponseError::Forbidden(e.to_string()), + access::Error::RulesError(error) => ResponseError::TooManyRequests(error), + access::Error::UnAuthenticated => ResponseError::Unauthorized, + _ => ResponseError::BadRequest(e.to_string()), + })?; + + Ok(true) +} diff --git a/sentry/src/routes/channel.rs b/sentry/src/routes/channel.rs index 7ad81a370..23f052c8d 100644 --- a/sentry/src/routes/channel.rs +++ b/sentry/src/routes/channel.rs @@ -178,52 +178,52 @@ pub async fn last_approved( .unwrap()) } -pub async fn insert_events( - req: Request, - app: &Application, -) -> Result, ResponseError> { - let (req_head, req_body) = req.into_parts(); - - let auth = req_head.extensions.get::(); - let session = req_head - .extensions - .get::() - .expect("request should have session"); - - let route_params = req_head - .extensions - .get::() - .expect("request should have route params"); - - let channel_id = ChannelId::from_hex(route_params.index(0))?; - - let body_bytes = hyper::body::to_bytes(req_body).await?; - let mut request_body = serde_json::from_slice::>>(&body_bytes)?; - - let events = request_body - .remove("events") - .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; - - // - // TODO #381: AIP#61 Spender Aggregator should be called - // - - // handle events - check access - // handle events - Update targeting rules - // calculate payout - // distribute fees - // handle spending - Spender Aggregate - // handle events - aggregate Events and put into analytics - - app.event_aggregator - .record(app, &channel_id, session, auth, events) - .await?; - - Ok(Response::builder() - .header("Content-type", "application/json") - .body(serde_json::to_string(&SuccessResponse { success: true })?.into()) - .unwrap()) -} +// pub async fn insert_events( +// req: Request, +// app: &Application, +// ) -> Result, ResponseError> { +// let (req_head, req_body) = req.into_parts(); + +// let auth = req_head.extensions.get::(); +// let session = req_head +// .extensions +// .get::() +// .expect("request should have session"); + +// let route_params = req_head +// .extensions +// .get::() +// .expect("request should have route params"); + +// let channel_id = ChannelId::from_hex(route_params.index(0))?; + +// let body_bytes = hyper::body::to_bytes(req_body).await?; +// let mut request_body = serde_json::from_slice::>>(&body_bytes)?; + +// let events = request_body +// .remove("events") +// .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; + +// // +// // TODO #381: AIP#61 Spender Aggregator should be called +// // + +// // handle events - check access +// // handle events - Update targeting rules +// // calculate payout +// // distribute fees +// // handle spending - Spender Aggregate +// // handle events - aggregate Events and put into analytics + +// // app.event_aggregator +// // .record(app, &channel_id, session, auth, events) +// // .await?; + +// Ok(Response::builder() +// .header("Content-type", "application/json") +// .body(serde_json::to_string(&SuccessResponse { success: true })?.into()) +// .unwrap()) +// } pub async fn create_validator_messages( req: Request, From 57116cc38237dd02c0dae2e34df0aabd3aadb2c7 Mon Sep 17 00:00:00 2001 From: simzzz Date: Tue, 15 Jun 2021 15:42:53 +0300 Subject: [PATCH 16/49] Accounting messages + First integration tests --- primitives/src/sentry/accounting.rs | 22 ++-- sentry/src/routes/campaign.rs | 160 +++++++++++++++++++++++----- 2 files changed, 146 insertions(+), 36 deletions(-) diff --git a/primitives/src/sentry/accounting.rs b/primitives/src/sentry/accounting.rs index 6c5b1b88d..9896db703 100644 --- a/primitives/src/sentry/accounting.rs +++ b/primitives/src/sentry/accounting.rs @@ -1,9 +1,6 @@ -use std::{ - convert::TryFrom, - marker::PhantomData, -}; +use std::{convert::TryFrom, marker::PhantomData}; -use crate::{balances_map::UnifiedMap, Address, channel_v5::Channel, UnifiedNum}; +use crate::{balances_map::UnifiedMap, channel_v5::Channel, Address, UnifiedNum}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Deserializer, Serialize}; use thiserror::Error; @@ -130,7 +127,8 @@ mod de { Ok(Self { channel: de_acc.channel, - balances: Balances::::try_from(de_acc.balances).map_err(serde::de::Error::custom)?, + balances: Balances::::try_from(de_acc.balances) + .map_err(serde::de::Error::custom)?, created: de_acc.created, updated: de_acc.updated, }) @@ -146,7 +144,10 @@ mod de { Ok(Self { channel: unchecked_acc.channel, - balances: unchecked_acc.balances.check().map_err(serde::de::Error::custom)?, + balances: unchecked_acc + .balances + .check() + .map_err(serde::de::Error::custom)?, created: unchecked_acc.created, updated: unchecked_acc.updated, }) @@ -204,13 +205,14 @@ mod postgres { impl TryFrom<&Row> for Accounting { type Error = Error; - + fn try_from(row: &Row) -> Result { let balances = Balances:: { earners: row.get::<_, Json<_>>("earners").0, spenders: row.get::<_, Json<_>>("spenders").0, state: PhantomData::default(), - }.check()?; + } + .check()?; Ok(Self { channel: row.get("channel"), @@ -220,4 +222,4 @@ mod postgres { }) } } -} \ No newline at end of file +} diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 4a10d9621..afd603aac 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -2,7 +2,7 @@ use crate::{ success_response, Application, ResponseError, db::{ spendable::fetch_spendable, - event_aggregate::latest_new_state_v5, + accounting::get_accounting_spent, DbPool }, }; @@ -12,11 +12,11 @@ use primitives::{ sentry::{ campaign_create::CreateCampaign, SuccessResponse, - MessageResponse, + accounting::Accounting, }, spender::Spendable, validator::NewState, - Campaign, CampaignId, UnifiedNum, BigNum + Address, Campaign, CampaignId, UnifiedNum, BigNum }; use redis::aio::MultiplexedConnection; use slog::error; @@ -43,13 +43,11 @@ pub async fn create_campaign( let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); - // Checking if there is enough remaining deposit - // TODO: Switch with Accounting once it's ready - let latest_new_state = latest_new_state_v5(&app.pool, &campaign.channel, "").await?; + let accounting_spent = get_accounting_spent(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; // TODO: AIP#61: Update when changes to Spendable are ready let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - let remaining_for_channel = get_total_remaining_for_channel(&app.redis, &app.pool, &campaign, &latest_new_state, &latest_spendable).await?; + let remaining_for_channel = get_total_remaining_for_channel(&campaign.creator, &accounting_spent, &latest_spendable)?; if campaign.budget > remaining_for_channel { return Err(ResponseError::Conflict("Not Enough budget for campaign".to_string())); @@ -147,24 +145,10 @@ async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: Campai Ok(true) } -async fn get_total_remaining_for_channel(redis: &MultiplexedConnection, pool: &DbPool, campaign: &Campaign, latest_new_state: &Option>, latest_spendable: &Spendable) -> Result { +fn get_total_remaining_for_channel(creator: &Address, accounting_spent: &UnifiedNum, latest_spendable: &Spendable) -> Result { let total_deposited = latest_spendable.deposit.total; - let latest_new_state = latest_new_state.as_ref().ok_or_else(|| ResponseError::Conflict("Error getting latest new state message".to_string()))?; - let msg = &latest_new_state.msg; - let total_spent = msg.balances.get(&campaign.creator); - let zero = BigNum::from(0); - let total_spent = if let Some(spent) = total_spent { - spent - } else { - &zero - }; - - // TODO: total_spent is BigNum, is it safe to just convert it to UnifiedNum like this? - let total_spent = total_spent.to_u64().ok_or_else(|| { - ResponseError::Conflict("Error while converting total_spent to u64".to_string()) - })?; - let total_remaining = total_deposited.checked_sub(&UnifiedNum::from_u64(total_spent)).ok_or_else(|| { + let total_remaining = total_deposited.checked_sub(&accounting_spent).ok_or_else(|| { ResponseError::Conflict("Error while calculating the total remaining amount".to_string()) })?; Ok(total_remaining) @@ -208,8 +192,7 @@ async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPoo pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { let campaign_spent = get_spent_for_campaign(&redis, campaign.id).await?; - // Getting the latest new state from Postgres - let latest_new_state = latest_new_state_v5(&pool, &campaign.channel, "").await?; + let accounting_spent = get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; // Check if we have reached the budget @@ -231,7 +214,7 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl update_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; // Gets the latest Spendable for this (spender, channelId) pair - let total_remaining = get_total_remaining_for_channel(&redis, &pool, &campaign, &latest_new_state, &latest_spendable).await?; + let total_remaining = get_total_remaining_for_channel(&campaign.creator, &accounting_spent, &latest_spendable)?; let campaigns_for_channel = get_campaigns_for_channel(&pool, &campaign).await?; let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &pool, &campaigns_for_channel, &campaign).await?; if campaigns_remaining_sum > total_remaining { @@ -244,3 +227,128 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 } + + + +#[cfg(test)] +mod test { + use primitives::{ + util::tests::prep_db::{DUMMY_CAMPAIGN, DUMMY_CHANNEL}, + Deposit + }; + + use deadpool::managed::Object; + + use crate::{ + db::redis_pool::{Manager, TESTS_POOL}, + }; + + use super::*; + + async fn get_redis() -> Object { + let connection = TESTS_POOL.get().await.expect("Should return Object"); + connection + } + + fn get_campaign() -> Campaign { + DUMMY_CAMPAIGN.clone() + } + + fn get_dummy_spendable(spender: Address) -> Spendable { + Spendable { + spender, + channel: DUMMY_CHANNEL.clone(), + deposit: Deposit { + total: UnifiedNum::from_u64(1_000_000), + still_on_create2: UnifiedNum::from_u64(0), + }, + } + } + + #[tokio::test] + async fn does_it_get_total_remaianing() { + let campaign = get_campaign(); + let accounting_spent = UnifiedNum::from_u64(100_000); + let latest_spendable = get_dummy_spendable(campaign.creator); + + let total_remaining = get_total_remaining_for_channel(&campaign.creator, &accounting_spent, &latest_spendable); + + assert_eq!(total_remaining, UnifiedNum::from_u64(900_000)); + } + + #[tokio::test] + async fn does_it_update_remaining() { + let redis = get_redis().await; + let campaign = get_campaign(); + let key = format!("remaining:{}", campaign.id); + + // Setting the redis base variable + redis::cmd("SET") + .arg(&key) + .arg(100u64) + .query_async(&mut redis.connection) + .await + .expect("should set"); + + // 2 async calls at once, should be 500 after them + futures::future::join( + update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)), + update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)) + ).await; + + let remaining = redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await + .expect("should get remaining"); + + assert_eq!(remaining, UnifiedNum::from_u64(500)); + + update_remaining_for_campaign(&redis, campaign.id, campaign.budget).await.expect("should increase"); + + let remaining = redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await + .expect("should get remaining"); + + let should_be_remaining = UnifiedNum::from_u64(500).checked_add(&campaign.budget).expect("should add"); + assert_eq!(remaining, should_be_remaining); + + update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(0)).await.expect("should work"); + + let remaining = redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await + .expect("should get remaining"); + + assert_eq!(remaining, should_be_remaining); + + update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(-500)).await.expect("should work"); + + let should_be_remaining = should_be_remaining.checked_sub(500).expect("should work"); + + let remaining = redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await + .expect("should get remaining"); + + assert_eq!(remaining, should_be_remaining); + } + + #[tokio::test] + async fn update_remaining_before_it_is_set() { + let redis = get_redis().await; + let campaign = get_campaign(); + let key = format!("remaining:{}", campaign.id); + + let remaining = redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(&mut redis.clone()) + .await; + + assert_eq!(remaining, Err(ResponseError::Conflict)) + } +} \ No newline at end of file From f4e0949edc0f4a2d400549738c2563b23ba65ca1 Mon Sep 17 00:00:00 2001 From: simzzz Date: Wed, 16 Jun 2021 12:03:48 +0300 Subject: [PATCH 17/49] fixed some unwraps --- sentry/src/routes/campaign.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index afd603aac..f48ff3d36 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -106,14 +106,16 @@ async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) - .await { Ok(Some(spent)) => { - let res = BigNum::from_str(&spent).unwrap(); - let res = res.to_u64().unwrap(); - UnifiedNum::from_u64(res) + let res = BigNum::from_str(&spent)?; + let res = res.to_u64().ok_or_else(|| { + ResponseError::Conflict("Error while converting BigNum to u64".to_string()) + })?; + Ok(UnifiedNum::from_u64(res)) }, - _ => UnifiedNum::from_u64(0) + _ => Ok(UnifiedNum::from_u64(0)) }; - Ok(campaign_spent) + campaign_spent } async fn get_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { @@ -124,13 +126,15 @@ async fn get_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignI .await { Ok(Some(remaining)) => { - let res = BigNum::from_str(&remaining).unwrap(); - let res = res.to_u64().unwrap(); - UnifiedNum::from_u64(res) + let res = BigNum::from_str(&remaining)?; + let res = res.to_u64().ok_or_else(|| { + ResponseError::Conflict("Error while calculating the total remaining amount".to_string()) + })?; + Ok(UnifiedNum::from_u64(res)) }, - _ => UnifiedNum::from_u64(0) + _ => Ok(UnifiedNum::from_u64(0)) }; - Ok(remaining) + remaining } async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { From 825d6cbeacee1c126de1f198e6a96350e55b2353 Mon Sep 17 00:00:00 2001 From: simzzz Date: Tue, 22 Jun 2021 11:06:31 +0300 Subject: [PATCH 18/49] more fixes --- sentry/src/db/campaign.rs | 33 ++++++---- sentry/src/db/event_aggregate.rs | 12 +--- sentry/src/lib.rs | 2 +- sentry/src/routes/campaign.rs | 109 +++++++++++++++---------------- 4 files changed, 78 insertions(+), 78 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index eba32944f..0d4ecaf8a 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -45,13 +45,17 @@ pub async fn fetch_campaign(pool: &DbPool, campaign: &Campaign) -> Result Result, PoolError> { +pub async fn get_campaigns_for_channel( + pool: &DbPool, + campaign: &Campaign, +) -> Result, PoolError> { let client = pool.get().await?; let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE channel_id = $1").await?; let rows = client.query(&statement, &[&campaign.channel.id()]).await?; let campaigns = rows.iter().map(Campaign::from).collect(); + Ok(campaigns) } @@ -75,16 +79,19 @@ pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result> 'type' = 'NewState' AND msg->> 'stateRoot' = $3 ORDER BY received DESC LIMIT 1").await?; let rows = client - .query( - &select, - &[ - &channel.id(), - &channel.leader, - &state_root, - ], - ) + .query(&select, &[&channel.id(), &channel.leader, &state_root]) .await?; rows.get(0) diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index b2dbd2634..bcfa5d940 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -33,9 +33,9 @@ use std::collections::HashMap; pub mod middleware; pub mod routes { pub mod analytics; + pub mod campaign; pub mod cfg; pub mod channel; - pub mod campaign; pub mod event_aggregate; pub mod validator_message; } diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index f48ff3d36..a8c2bca7d 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -11,12 +11,9 @@ use primitives::{ adapter::Adapter, sentry::{ campaign_create::CreateCampaign, - SuccessResponse, - accounting::Accounting, }, spender::Spendable, - validator::NewState, - Address, Campaign, CampaignId, UnifiedNum, BigNum + Campaign, CampaignId, UnifiedNum, BigNum }; use redis::aio::MultiplexedConnection; use slog::error; @@ -47,7 +44,7 @@ pub async fn create_campaign( // TODO: AIP#61: Update when changes to Spendable are ready let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - let remaining_for_channel = get_total_remaining_for_channel(&campaign.creator, &accounting_spent, &latest_spendable)?; + let remaining_for_channel = get_total_remaining_for_channel(&accounting_spent, &latest_spendable)?; if campaign.budget > remaining_for_channel { return Err(ResponseError::Conflict("Not Enough budget for campaign".to_string())); @@ -91,8 +88,6 @@ pub async fn update_campaign( _ => Ok(()), }?; - let update_response = SuccessResponse { success: true }; - Ok(success_response(serde_json::to_string(&campaign)?)) } @@ -118,7 +113,7 @@ async fn get_spent_for_campaign(redis: &MultiplexedConnection, id: CampaignId) - campaign_spent } -async fn get_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result { +async fn get_remaining_for_campaign_from_redis(redis: &MultiplexedConnection, id: CampaignId) -> Result { let key = format!("remaining:{}", id); let remaining = match redis::cmd("GET") .arg(&key) @@ -137,6 +132,7 @@ async fn get_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignI remaining } +// tested async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { // update a key in Redis for the remaining spendable amount let key = format!("remaining:{}", id); @@ -149,7 +145,8 @@ async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: Campai Ok(true) } -fn get_total_remaining_for_channel(creator: &Address, accounting_spent: &UnifiedNum, latest_spendable: &Spendable) -> Result { +// tested +fn get_total_remaining_for_channel(accounting_spent: &UnifiedNum, latest_spendable: &Spendable) -> Result { let total_deposited = latest_spendable.deposit.total; let total_remaining = total_deposited.checked_sub(&accounting_spent).ok_or_else(|| { @@ -158,31 +155,29 @@ fn get_total_remaining_for_channel(creator: &Address, accounting_spent: &Unified Ok(total_remaining) } -// async fn update_remaining_for_channel(redis: &MultiplexedConnection, id: ChannelId, amount: UnifiedNum) -> Result { -// let key = format!("adexChannel:remaining:{}", id); -// redis::cmd("SET") -// .arg(&key) -// .arg(amount.to_u64()) -// .query_async(&mut redis.clone()) -// .await?; -// Ok(true) -// } - -async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPool, campaigns: &Vec, mutated_campaign: &Campaign) -> Result { +async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign_id: CampaignId) -> Result, ResponseError> { let other_campaigns_remaining = campaigns .into_iter() - .filter(|c| c.id != mutated_campaign.id) + .filter(|c| c.id != mutated_campaign_id) .map(|c| async move { let spent = get_spent_for_campaign(&redis, c.id).await?; - let remaining = c.budget.checked_sub(&spent)?; + let remaining = c.budget.checked_sub(&spent).ok_or_else(|| { + ResponseError::Conflict("Error while calculating remaining for mutated campaign".to_string()) + })?; Ok(remaining) }) .collect::>(); let other_campaigns_remaining = join_all(other_campaigns_remaining).await; - // TODO: Fix Unwrap + let other_campaigns_remaining: Result, _> = other_campaigns_remaining.into_iter().collect(); + other_campaigns_remaining +} + + +async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: &Campaign) -> Result { + let other_campaigns_remaining = get_remaining_for_multiple_campaigns(&redis, &campaigns, mutated_campaign.id).await?; let sum_of_campaigns_remaining = other_campaigns_remaining .into_iter() - .fold(UnifiedNum::from_u64(0), |mut sum, val| sum.checked_add(&val).unwrap()); + .try_fold(UnifiedNum::from_u64(0), |sum, val| sum.checked_add(&val).ok_or(ResponseError::Conflict("Couldn't sum remaining for campaigns".to_string())))?; // Necessary to do it explicitly for current campaign as its budget is not yet updated in DB let spent_for_mutated_campaign = get_spent_for_campaign(&redis, mutated_campaign.id).await?; let remaining_for_mutated_campaign = mutated_campaign.budget.checked_sub(&spent_for_mutated_campaign).ok_or_else(|| { @@ -190,7 +185,7 @@ async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, pool: &DbPoo })?; sum_of_campaigns_remaining.checked_add(&remaining_for_mutated_campaign).ok_or_else(|| { ResponseError::Conflict("Error while calculating sum for all campaigns".to_string()) - }); + })?; Ok(sum_of_campaigns_remaining) } @@ -204,7 +199,7 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl return Err(ResponseError::FailedValidation("No more budget available for spending".into())); } - let old_remaining = get_remaining_for_campaign(&redis, campaign.id).await?; + let old_remaining = get_remaining_for_campaign_from_redis(&redis, campaign.id).await?; let old_remaining = UnifiedNum::from_u64(max(0, old_remaining.to_u64())); let new_remaining = campaign.budget.checked_sub(&campaign_spent).ok_or_else(|| { @@ -218,9 +213,9 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl update_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; // Gets the latest Spendable for this (spender, channelId) pair - let total_remaining = get_total_remaining_for_channel(&campaign.creator, &accounting_spent, &latest_spendable)?; + let total_remaining = get_total_remaining_for_channel(&accounting_spent, &latest_spendable)?; let campaigns_for_channel = get_campaigns_for_channel(&pool, &campaign).await?; - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &pool, &campaigns_for_channel, &campaign).await?; + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, &campaign).await?; if campaigns_remaining_sum > total_remaining { return Err(ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel".to_string())); } @@ -237,16 +232,15 @@ pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &Multipl #[cfg(test)] mod test { use primitives::{ - util::tests::prep_db::{DUMMY_CAMPAIGN, DUMMY_CHANNEL}, - Deposit + util::tests::prep_db::{DUMMY_CAMPAIGN}, + spender::Deposit, + Address }; - use deadpool::managed::Object; - use crate::{ db::redis_pool::{Manager, TESTS_POOL}, }; - + use std::convert::TryFrom; use super::*; async fn get_redis() -> Object { @@ -261,7 +255,7 @@ mod test { fn get_dummy_spendable(spender: Address) -> Spendable { Spendable { spender, - channel: DUMMY_CHANNEL.clone(), + channel: DUMMY_CAMPAIGN.channel.clone(), deposit: Deposit { total: UnifiedNum::from_u64(1_000_000), still_on_create2: UnifiedNum::from_u64(0), @@ -275,14 +269,14 @@ mod test { let accounting_spent = UnifiedNum::from_u64(100_000); let latest_spendable = get_dummy_spendable(campaign.creator); - let total_remaining = get_total_remaining_for_channel(&campaign.creator, &accounting_spent, &latest_spendable); + let total_remaining = get_total_remaining_for_channel(&accounting_spent, &latest_spendable).expect("should calculate"); assert_eq!(total_remaining, UnifiedNum::from_u64(900_000)); } #[tokio::test] async fn does_it_update_remaining() { - let redis = get_redis().await; + let mut redis = get_redis().await; let campaign = get_campaign(); let key = format!("remaining:{}", campaign.id); @@ -290,7 +284,7 @@ mod test { redis::cmd("SET") .arg(&key) .arg(100u64) - .query_async(&mut redis.connection) + .query_async::<_, ()>(&mut redis.connection) .await .expect("should set"); @@ -302,20 +296,26 @@ mod test { let remaining = redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) + .query_async::<_, Option>(&mut redis.connection) .await .expect("should get remaining"); + assert_eq!(remaining.is_some(), true); + let remaining = remaining.expect("should get remaining"); + let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); assert_eq!(remaining, UnifiedNum::from_u64(500)); update_remaining_for_campaign(&redis, campaign.id, campaign.budget).await.expect("should increase"); let remaining = redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) + .query_async::<_, Option>(&mut redis.connection) .await .expect("should get remaining"); + assert_eq!(remaining.is_some(), true); + let remaining = remaining.expect("should get remaining"); + let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); let should_be_remaining = UnifiedNum::from_u64(500).checked_add(&campaign.budget).expect("should add"); assert_eq!(remaining, should_be_remaining); @@ -323,36 +323,35 @@ mod test { let remaining = redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) - .await - .expect("should get remaining"); - - assert_eq!(remaining, should_be_remaining); - - update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(-500)).await.expect("should work"); - - let should_be_remaining = should_be_remaining.checked_sub(500).expect("should work"); - - let remaining = redis::cmd("GET") - .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) + .query_async::<_, Option>(&mut redis.connection) .await .expect("should get remaining"); + assert_eq!(remaining.is_some(), true); + let remaining = remaining.expect("should get remaining"); + let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); assert_eq!(remaining, should_be_remaining); } #[tokio::test] async fn update_remaining_before_it_is_set() { - let redis = get_redis().await; + let mut redis = get_redis().await; let campaign = get_campaign(); let key = format!("remaining:{}", campaign.id); let remaining = redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) + .query_async::<_, Option>(&mut redis.connection) .await; - assert_eq!(remaining, Err(ResponseError::Conflict)) + assert_eq!(remaining.is_err(), true) } + + // test get_campaigns_remaining_sum + + // test get_remaining_for_multiple_campaigns + + // test get_remaining_for_campaign_from_redis + + // test get_spent_for_campaign } \ No newline at end of file From 5f0e1c9adca595459bc4c5f8e28b2aa541d11135 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 22 Jun 2021 17:47:30 +0300 Subject: [PATCH 19/49] sentry - routes - channel - remove insert_events --- sentry/src/routes/channel.rs | 47 ------------------------------------ 1 file changed, 47 deletions(-) diff --git a/sentry/src/routes/channel.rs b/sentry/src/routes/channel.rs index 23f052c8d..304bd3432 100644 --- a/sentry/src/routes/channel.rs +++ b/sentry/src/routes/channel.rs @@ -178,53 +178,6 @@ pub async fn last_approved( .unwrap()) } -// pub async fn insert_events( -// req: Request, -// app: &Application, -// ) -> Result, ResponseError> { -// let (req_head, req_body) = req.into_parts(); - -// let auth = req_head.extensions.get::(); -// let session = req_head -// .extensions -// .get::() -// .expect("request should have session"); - -// let route_params = req_head -// .extensions -// .get::() -// .expect("request should have route params"); - -// let channel_id = ChannelId::from_hex(route_params.index(0))?; - -// let body_bytes = hyper::body::to_bytes(req_body).await?; -// let mut request_body = serde_json::from_slice::>>(&body_bytes)?; - -// let events = request_body -// .remove("events") -// .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; - -// // -// // TODO #381: AIP#61 Spender Aggregator should be called -// // - -// // handle events - check access -// // handle events - Update targeting rules -// // calculate payout -// // distribute fees -// // handle spending - Spender Aggregate -// // handle events - aggregate Events and put into analytics - -// // app.event_aggregator -// // .record(app, &channel_id, session, auth, events) -// // .await?; - -// Ok(Response::builder() -// .header("Content-type", "application/json") -// .body(serde_json::to_string(&SuccessResponse { success: true })?.into()) -// .unwrap()) -// } - pub async fn create_validator_messages( req: Request, app: &Application, From 115b7b0bef98e4398ed0e17b2e34021e536b557d Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 23 Jun 2021 12:09:51 +0300 Subject: [PATCH 20/49] sentry - clean up Events code in Channel & access --- primitives/src/sentry.rs | 3 +-- sentry/src/access.rs | 22 +++++----------------- sentry/src/payout.rs | 2 +- sentry/src/routes/channel.rs | 4 ++-- 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index afd8a6214..dbfc64740 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -1,7 +1,6 @@ use crate::{ - targeting::Rules, validator::{ApproveState, Heartbeat, MessageTypes, NewState, Type as MessageType}, - Address, BalancesMap, BigNum, Channel, ChannelId, ValidatorId, IPFS, + Address, BigNum, Channel, ChannelId, ValidatorId, IPFS, }; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; diff --git a/sentry/src/access.rs b/sentry/src/access.rs index cc49caf7c..a3fca1578 100644 --- a/sentry/src/access.rs +++ b/sentry/src/access.rs @@ -13,7 +13,7 @@ use thiserror::Error; #[derive(Debug, PartialEq, Eq, Error)] pub enum Error { - #[error("channel is expired")] + #[error("Campaign is expired")] CampaignIsExpired, #[error("event submission restricted")] ForbiddenReferrer, @@ -37,14 +37,7 @@ pub async fn check_access( if current_time > campaign.active.to { return Err(Error::CampaignIsExpired); } - - let (is_creator, auth_uid) = match auth { - Some(auth) => ( - auth.uid.to_address() == campaign.creator, - auth.uid.to_string(), - ), - None => (false, Default::default()), - }; + let auth_uid = auth.map(|auth| auth.uid.to_string()).unwrap_or_default(); // Rules for events if forbidden_country(&session) || forbidden_referrer(&session) { @@ -94,11 +87,7 @@ pub async fn check_access( ) })); - if let Err(rule_error) = apply_all_rules.await { - Err(Error::RulesError(rule_error)) - } else { - Ok(()) - } + apply_all_rules.await.map_err(Error::RulesError).map(|_| ()) } async fn apply_rule( @@ -188,9 +177,8 @@ mod test { config::configuration, event_submission::{RateLimit, Rule}, sentry::Event, - targeting::Rules, - util::tests::prep_db::{ADDRESSES, DUMMY_CAMPAIGN, DUMMY_CHANNEL, IDS}, - Channel, Config, EventSubmission, + util::tests::prep_db::{ADDRESSES, DUMMY_CAMPAIGN, IDS}, + Config, EventSubmission, }; use deadpool::managed::Object; diff --git a/sentry/src/payout.rs b/sentry/src/payout.rs index e0ad94930..29373aa28 100644 --- a/sentry/src/payout.rs +++ b/sentry/src/payout.rs @@ -11,6 +11,7 @@ use std::cmp::{max, min}; pub type Result = std::result::Result, Error>; +/// If None is returned this means that the targeting rules evaluation has set `show = false` pub fn get_payout( logger: &Logger, campaign: &Campaign, @@ -88,7 +89,6 @@ pub fn get_payout( } } } - _ => Ok(None), } } diff --git a/sentry/src/routes/channel.rs b/sentry/src/routes/channel.rs index 304bd3432..ca337b6f7 100644 --- a/sentry/src/routes/channel.rs +++ b/sentry/src/routes/channel.rs @@ -3,7 +3,7 @@ use crate::db::{ get_channel_by_id, insert_channel, insert_validator_messages, list_channels, update_exhausted_channel, PoolError, }; -use crate::{success_response, Application, Auth, ResponseError, RouteParams, Session}; +use crate::{success_response, Application, Auth, ResponseError, RouteParams}; use futures::future::try_join_all; use hex::FromHex; use hyper::{Body, Request, Response}; @@ -11,7 +11,7 @@ use primitives::{ adapter::Adapter, sentry::{ channel_list::{ChannelListQuery, LastApprovedQuery}, - Event, LastApproved, LastApprovedResponse, SuccessResponse, + LastApproved, LastApprovedResponse, SuccessResponse, }, validator::MessageTypes, Channel, ChannelId, From 067b3afff269be3044fcc2c3ae7d9ad01f7f169c Mon Sep 17 00:00:00 2001 From: simzzz Date: Tue, 29 Jun 2021 18:18:17 +0300 Subject: [PATCH 21/49] fixed remaining issues in PR --- primitives/src/sentry.rs | 37 +++-- sentry/src/db/campaign.rs | 41 +++-- sentry/src/db/event_aggregate.rs | 19 --- sentry/src/lib.rs | 24 ++- sentry/src/routes/campaign.rs | 263 ++++++++++++++++++------------- sentry/src/routes/channel.rs | 4 +- 6 files changed, 218 insertions(+), 170 deletions(-) diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index 0a5edf6e7..cb2fe99c2 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -1,7 +1,6 @@ use crate::{ - targeting::Rules, validator::{ApproveState, Heartbeat, MessageTypes, NewState, Type as MessageType}, - Address, BalancesMap, BigNum, Channel, ChannelId, ValidatorId, IPFS, + Address, BigNum, Channel, ChannelId, ValidatorId, IPFS, }; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -367,16 +366,30 @@ pub mod campaign_create { } // All editable fields stored in one place, used for checking when a budget is changed - // #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] - // pub struct CampaignState { - // pub budget: UnifiedNum, - // pub validators: Validators, - // pub title: Option, - // pub pricing_bounds: Option, - // pub event_submission: Option, - // pub ad_units: Vec, - // pub targeting_rules: Rules, - // } + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] + pub struct ModifyCampaign { + pub budget: Option, + pub validators: Option, + pub title: Option, + pub pricing_bounds: Option, + pub event_submission: Option, + pub ad_units: Option>, + pub targeting_rules: Option, + } + + impl ModifyCampaign { + pub fn from_campaign(campaign: Campaign) -> Self { + ModifyCampaign { + budget: Some(campaign.budget), + validators: Some(campaign.validators), + title: campaign.title, + pricing_bounds: campaign.pricing_bounds, + event_submission: campaign.event_submission, + ad_units: Some(campaign.ad_units), + targeting_rules: Some(campaign.targeting_rules), + } + } + } } #[cfg(feature = "postgres")] diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 49f2dca11..4f4817e72 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -1,5 +1,5 @@ use crate::db::{DbPool, PoolError}; -use primitives::{Campaign, CampaignId}; +use primitives::{ChannelId, CampaignId, Campaign}; use tokio_postgres::types::Json; pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result { @@ -48,32 +48,20 @@ pub async fn fetch_campaign( Ok(row.as_ref().map(Campaign::from)) } -pub async fn get_campaigns_for_channel( +pub async fn get_campaigns_by_channel( pool: &DbPool, - campaign: &Campaign, + channel_id: &ChannelId, ) -> Result, PoolError> { let client = pool.get().await?; let statement = client.prepare("SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to FROM campaigns WHERE channel_id = $1").await?; - let rows = client.query(&statement, &[&campaign.channel.id()]).await?; + let rows = client.query(&statement, &[&channel_id]).await?; let campaigns = rows.iter().map(Campaign::from).collect(); Ok(campaigns) } -pub async fn campaign_exists(pool: &DbPool, campaign: &Campaign) -> Result { - let client = pool.get().await?; - let statement = client - .prepare("SELECT EXISTS(SELECT 1 FROM campaigns WHERE id = $1)") - .await?; - - let row = client.execute(&statement, &[&campaign.id]).await?; - - let exists = row == 1; - Ok(exists) -} - // TODO: Test for campaign ad_units pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; @@ -104,8 +92,12 @@ pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result { + assert!(true); + } + _ => assert!(false), + } let fetched_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing.id) .await diff --git a/sentry/src/db/event_aggregate.rs b/sentry/src/db/event_aggregate.rs index 106a47ccc..73b73d138 100644 --- a/sentry/src/db/event_aggregate.rs +++ b/sentry/src/db/event_aggregate.rs @@ -1,7 +1,6 @@ use chrono::{DateTime, Utc}; use futures::pin_mut; use primitives::{ - channel_v5::Channel as ChannelV5, sentry::{EventAggregate, MessageResponse}, validator::{ApproveState, Heartbeat, NewState}, Address, BigNum, Channel, ChannelId, ValidatorId, @@ -59,24 +58,6 @@ pub async fn latest_new_state( .map_err(PoolError::Backend) } -pub async fn latest_new_state_v5( - pool: &DbPool, - channel: &ChannelV5, - state_root: &str, -) -> Result>, PoolError> { - let client = pool.get().await?; - - let select = client.prepare("SELECT \"from\", msg, received FROM validator_messages WHERE channel_id = $1 AND \"from\" = $2 AND msg ->> 'type' = 'NewState' AND msg->> 'stateRoot' = $3 ORDER BY received DESC LIMIT 1").await?; - let rows = client - .query(&select, &[&channel.id(), &channel.leader, &state_root]) - .await?; - - rows.get(0) - .map(MessageResponse::::try_from) - .transpose() - .map_err(PoolError::Backend) -} - pub async fn latest_heartbeats( pool: &DbPool, channel_id: &ChannelId, diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index d9e13d387..842799c5d 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::all)] #![deny(rust_2018_idioms)] -use crate::db::DbPool; +use crate::db::{DbPool, fetch_campaign}; use crate::routes::campaign; use crate::routes::channel::channel_status; use crate::routes::event_aggregate::list_channel_event_aggregates; @@ -18,17 +18,18 @@ use middleware::{campaign::CampaignLoad, Chain, Middleware}; use once_cell::sync::Lazy; use primitives::adapter::Adapter; use primitives::sentry::ValidationErrorResponse; -use primitives::{Config, ValidatorId}; +use primitives::{Config, ValidatorId, CampaignId}; use redis::aio::MultiplexedConnection; use regex::Regex; use routes::analytics::{advanced_analytics, advertiser_analytics, analytics, publisher_analytics}; -use routes::campaign::{create_campaign, update_campaign}; +use routes::campaign::{create_campaign, update_campaign::handle_route}; use routes::cfg::config; use routes::channel::{ channel_list, channel_validate, create_channel, create_validator_messages, last_approved, }; use slog::Logger; use std::collections::HashMap; +use std::str::FromStr; pub mod middleware; pub mod routes { @@ -63,7 +64,7 @@ lazy_static! { static ref PUBLISHER_ANALYTICS_BY_CHANNEL_ID: Regex = Regex::new(r"^/analytics/for-publisher/0x([a-zA-Z0-9]{64})/?$").expect("The regex should be valid"); static ref CREATE_EVENTS_BY_CHANNEL_ID: Regex = Regex::new(r"^/channel/0x([a-zA-Z0-9]{64})/events/?$").expect("The regex should be valid"); static ref CAMPAIGN_UPDATE_BY_ID: Regex = - Regex::new(r"^/channel/0x([a-zA-Z0-9]{64})/?$").expect("The regex should be valid"); + Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/?$").expect("The regex should be valid"); } static INSERT_EVENTS_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { @@ -194,7 +195,20 @@ async fn campaigns_router( let (path, method) = (req.uri().path(), req.method()); // create events - if let (Some(caps), &Method::POST) = (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) { + if let (Some(caps), &Method::POST) = (CAMPAIGN_UPDATE_BY_ID.captures(&path), method) { + let param = RouteParams(vec![caps + .get(1) + .map_or("".to_string(), |m| m.as_str().to_string())]); + + // Should be safe to access indice here + let campaign_id = param.get(0).ok_or_else(|| ResponseError::BadRequest("No CampaignId".to_string()))?; + let campaign_id = CampaignId::from_str(&campaign_id).map_err(|_| ResponseError::BadRequest("Bad CampaignId".to_string()))?; + + let campaign = fetch_campaign(app.pool.clone(), &campaign_id).await.map_err(|_| ResponseError::NotFound)?; + + req.extensions_mut().insert(campaign); + handle_route(req, app).await + } else if let (Some(caps), &Method::POST) = (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) { let param = RouteParams(vec![caps .get(1) .map_or("".to_string(), |m| m.as_str().to_string())]); diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index e396200f6..f7e08f8e0 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -3,40 +3,56 @@ use crate::{ db::{ spendable::fetch_spendable, accounting::get_accounting_spent, + campaign::{update_campaign, insert_campaign, get_campaigns_by_channel}, DbPool }, + routes::campaign::update_campaign::increase_remaining_for_campaign, + access::{self, check_access}, + Auth, Session }; + use hyper::{Body, Request, Response}; use primitives::{ adapter::Adapter, sentry::{ - campaign_create::CreateCampaign, + campaign_create::{CreateCampaign, ModifyCampaign}, + Event, SuccessResponse, }, spender::Spendable, Campaign, CampaignId, UnifiedNum, BigNum }; use redis::aio::MultiplexedConnection; use slog::error; -use crate::db::campaign::{update_campaign as update_campaign_db, insert_campaign, get_campaigns_for_channel}; use std::{ cmp::max, str::FromStr, }; use futures::future::join_all; use std::collections::HashMap; +use deadpool_postgres::PoolError; +use tokio_postgres::error::SqlState; +use redis::RedisError; -use crate::{ - access::{self, check_access}, - success_response, Application, Auth, ResponseError, Session, -}; use chrono::Utc; -use hyper::{Body, Request, Response}; -use primitives::{ - adapter::Adapter, - sentry::{campaign_create::CreateCampaign, Event, SuccessResponse}, - Campaign, -}; -use crate::routes::campaign::modify_campaign::{get_remaining_for_campaign_from_redis, get_campaigns_remaining_sum}; + +#[derive(Debug, PartialEq, Eq)] +pub enum CampaignError { + FailedUpdate(String), + CalculationError, + BudgetExceeded, +} + +impl From for CampaignError { + fn from(err: RedisError) -> Self { + CampaignError::FailedUpdate(err.to_string()) + } +} + +impl From for CampaignError { + fn from(err: PoolError) -> Self { + CampaignError::FailedUpdate(err.to_string()) + } +} pub async fn create_campaign( req: Request, @@ -57,136 +73,166 @@ pub async fn create_campaign( // TODO: AIP#61: Update when changes to Spendable are ready let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - let remaining_for_channel = get_total_remaining_for_channel(&accounting_spent, &latest_spendable)?; + let remaining_for_channel = get_total_remaining_for_channel(&accounting_spent, &latest_spendable).ok_or(ResponseError::BadRequest("couldn't get total remaining for channel".to_string()))?; if campaign.budget > remaining_for_channel { - return Err(ResponseError::Conflict("Not Enough budget for campaign".to_string())); + return Err(ResponseError::BadRequest("Not Enough budget for campaign".to_string())); } // If the channel is being created, the amount spent is 0, therefore remaining = budget - update_remaining_for_campaign(&app.redis.clone(), campaign.id, campaign.budget).await?; + increase_remaining_for_campaign(&app.redis.clone(), campaign.id, campaign.budget).await.map_err(|_| ResponseError::BadRequest("Couldn't update remaining while creating campaign".to_string()))?; // insert Campaign match insert_campaign(&app.pool, &campaign).await { Err(error) => { error!(&app.logger, "{}", &error; "module" => "create_campaign"); - return Err(ResponseError::Conflict("campaign already exists".to_string())); + match error { + PoolError::Backend(error) if error.code() == Some(&SqlState::UNIQUE_VIOLATION) => { + Err(ResponseError::Conflict( + "Campaign already exists".to_string(), + )) + } + _ => Err(error_response), + } } - Ok(false) => Err(error_response), + Ok(false) => Err(ResponseError::BadRequest("Encountered error while creating Campaign; please try again".to_string())), _ => Ok(()), }?; Ok(success_response(serde_json::to_string(&campaign)?)) } -pub async fn update_campaign( - req: Request, - app: &Application, -) -> Result, ResponseError> { - let body = hyper::body::to_bytes(req.into_body()).await?; - let campaign = serde_json::from_slice::(&body) - .map_err(|e| ResponseError::FailedValidation(e.to_string()))?; +// tested +fn get_total_remaining_for_channel(accounting_spent: &UnifiedNum, latest_spendable: &Spendable) -> Option { + let total_deposited = latest_spendable.deposit.total; - let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); + let total_remaining = total_deposited.checked_sub(&accounting_spent); + total_remaining +} - // modify Campaign +pub mod update_campaign { + use super::*; + use lazy_static::lazy_static; - match modify_campaign(&app.pool, &campaign, &app.redis).await { - Err(error) => { - error!(&app.logger, "{:?}", &error; "module" => "update_campaign"); - return Err(ResponseError::Conflict("Error modifying campaign".to_string())); - } - Ok(false) => Err(error_response), - _ => Ok(()), - }?; + lazy_static!{ + pub static ref CAMPAIGN_REMAINING_KEY: &'static str = "campaignRemaining"; + } - Ok(success_response(serde_json::to_string(&campaign)?)) -} + pub async fn increase_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, id); + redis::cmd("INCRBY") + .arg(&key) + .arg(amount.to_u64()) + .query_async(&mut redis.clone()) + .await?; + Ok(true) + } -pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, redis: &MultiplexedConnection) -> Result { - let accounting_spent = get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + pub async fn decrease_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, id); + redis::cmd("DECRBY") + .arg(&key) + .arg(amount.to_u64()) + .query_async(&mut redis.clone()) + .await?; + Ok(true) + } - let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + pub async fn handle_route( + req: Request, + app: &Application, + ) -> Result, ResponseError> { + let campaign = req.extensions().get::().expect("We must have a campaign in extensions"); - let old_remaining = get_remaining_for_campaign_from_redis(&redis, campaign.id).await?; - let campaign_spent = campaign.budget.checked_sub(&old_remaining)?; + let modified_campaign = ModifyCampaign::from_campaign(campaign.clone()); - let old_remaining = UnifiedNum::from_u64(max(0, old_remaining.to_u64())); + let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); - let new_remaining = campaign.budget.checked_sub(&campaign_spent).ok_or_else(|| { - ResponseError::Conflict("Error while subtracting campaign_spent from budget".to_string()) - })?; + // modify Campaign + modify_campaign(&app.pool, &campaign, &modified_campaign, &app.redis).await.map_err(|_| ResponseError::BadRequest("Failed to update campaign".to_string()))?; - let diff_in_remaining = new_remaining.checked_sub(&old_remaining).ok_or_else(|| { - ResponseError::Conflict("Error while subtracting campaign_spent from budget".to_string()) - })?; + Ok(success_response(serde_json::to_string(&campaign)?)) + } - update_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; + pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, modified_campaign: &ModifyCampaign, redis: &MultiplexedConnection) -> Result { + // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent + // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] + // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 + let accounting_spent = get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - // Gets the latest Spendable for this (spender, channelId) pair - let total_remaining = get_total_remaining_for_channel(&accounting_spent, &latest_spendable)?; - let campaigns_for_channel = get_campaigns_for_channel(&pool, &campaign).await?; - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, &campaign).await?; - if campaigns_remaining_sum > total_remaining { - return Err(ResponseError::Conflict("Remaining for campaigns exceeds total remaining for channel".to_string())); - } + let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - update_campaign_db(&pool, &campaign).await.map_err(|e| ResponseError::Conflict(e.to_string())) + let old_remaining = get_remaining_for_campaign_from_redis(&redis, campaign.id).await.ok_or(CampaignError::FailedUpdate("Couldn't get remaining for campaign".to_string()))?; - // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent - // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] - // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 -} + let campaign_spent = campaign.budget.checked_sub(&old_remaining).ok_or(CampaignError::CalculationError)?; + if campaign_spent >= campaign.budget { + return Err(CampaignError::BudgetExceeded); + } -// tested -async fn update_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { - // update a key in Redis for the remaining spendable amount - let key = format!("remaining:{}", id); - redis::cmd("INCRBY") - .arg(&key) - .arg(amount.to_u64()) - .query_async(&mut redis.clone()) - .await - .map_err(|_| ResponseError::Conflict("Error updating remainingSpendable for current campaign".to_string()))?; - Ok(true) -} + let old_remaining = UnifiedNum::from_u64(max(0, old_remaining.to_u64())); -// tested -fn get_total_remaining_for_channel(accounting_spent: &UnifiedNum, latest_spendable: &Spendable) -> Option { - let total_deposited = latest_spendable.deposit.total; + let new_remaining = campaign.budget.checked_sub(&campaign_spent).ok_or(CampaignError::CalculationError)?; - let total_remaining = total_deposited.checked_sub(&accounting_spent); - total_remaining -} + if new_remaining >= old_remaining { + let diff_in_remaining = new_remaining.checked_sub(&old_remaining).ok_or(CampaignError::CalculationError)?; + increase_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; + } else { + let diff_in_remaining = old_remaining.checked_sub(&new_remaining).ok_or(CampaignError::CalculationError)?; + decrease_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; + } -mod modify_campaign { - use super::*; - pub async fn get_remaining_for_campaign_from_redis(redis: &MultiplexedConnection, id: CampaignId) -> Result { - let key = format!("remaining:{}", id); + + // Gets the latest Spendable for this (spender, channelId) pair + let total_remaining = get_total_remaining_for_channel(&accounting_spent, &latest_spendable).ok_or(CampaignError::FailedUpdate("Could not get total remaining for channel".to_string()))?; + let campaigns_for_channel = get_campaigns_by_channel(&pool, &campaign.channel.id()).await?; + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, &campaign).await.map_err(|_| CampaignError::CalculationError)?; + if campaigns_remaining_sum > total_remaining { + return Err(CampaignError::BudgetExceeded); + } + + update_campaign(&pool, &campaign).await?; + + Ok(campaign.clone()) + } + + // TODO: #382 Remove after #412 is merged + fn get_unified_num_from_string(value: &str) -> Option { + let value_as_big_num: Option = BigNum::from_str(value).ok(); + let value_as_u64 = match value_as_big_num { + Some(num) => num.to_u64(), + _ => None, + }; + let value_as_unified = match value_as_u64 { + Some(num) => Some(UnifiedNum::from_u64(num)), + _ => None + }; + value_as_unified + } + + pub async fn get_remaining_for_campaign_from_redis(redis: &MultiplexedConnection, id: CampaignId) -> Option { + let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, id); let remaining = match redis::cmd("GET") .arg(&key) .query_async::<_, Option>(&mut redis.clone()) .await { Ok(Some(remaining)) => { - let res = BigNum::from_str(&remaining)?; - let res = res.to_u64().ok_or_else(|| { - ResponseError::Conflict("Error while calculating the total remaining amount".to_string()) - })?; - Ok(UnifiedNum::from_u64(res)) + // TODO: #382 Just parse from string once #412 is merged + get_unified_num_from_string(&remaining) }, - _ => Ok(UnifiedNum::from_u64(0)) + _ => None }; remaining } - async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign_id: CampaignId) -> Result, ResponseError> { + async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign_id: CampaignId) -> Result, CampaignError> { let other_campaigns_remaining = campaigns .into_iter() .filter(|c| c.id != mutated_campaign_id) + // TODO: Do 1 call with MGET .map(|c| async move { - let remaining = get_remaining_for_campaign_from_redis(&redis, c.id).await?; + let remaining = get_remaining_for_campaign_from_redis(&redis, c.id).await.ok_or(CampaignError::FailedUpdate("Couldn't get remaining for campaign".to_string()))?; Ok(remaining) }) .collect::>(); @@ -195,21 +241,17 @@ mod modify_campaign { other_campaigns_remaining } - pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: &Campaign) -> Result { + pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: &Campaign) -> Result { let other_campaigns_remaining = get_remaining_for_multiple_campaigns(&redis, &campaigns, mutated_campaign.id).await?; let sum_of_campaigns_remaining = other_campaigns_remaining .into_iter() - .try_fold(UnifiedNum::from_u64(0), |sum, val| sum.checked_add(&val).ok_or(ResponseError::Conflict("Couldn't sum remaining for campaigns".to_string())))?; + .try_fold(UnifiedNum::from_u64(0), |sum, val| sum.checked_add(&val).ok_or(CampaignError::CalculationError))?; // Necessary to do it explicitly for current campaign as its budget is not yet updated in DB - let old_remaining_for_mutated_campaign = get_remaining_for_campaign_from_redis(&redis, mutated_campaign.id); - let spent_for_mutated_campaign = mutated_campaign.budget.checked_sub(old_remaining_for_mutated_campaign); - let new_remaining_for_mutated_campaign = mutated_campaign.budget.checked_sub(&spent_for_mutated_campaign).ok_or_else(|| { - ResponseError::Conflict("Error while calculating remaining for mutated campaign".to_string()) - })?; - sum_of_campaigns_remaining.checked_add(&new_remaining_for_mutated_campaign).ok_or_else(|| { - ResponseError::Conflict("Error while calculating sum for all campaigns".to_string()) - })?; + let old_remaining_for_mutated_campaign = get_remaining_for_campaign_from_redis(&redis, mutated_campaign.id).await.ok_or(CampaignError::CalculationError)?; + let spent_for_mutated_campaign = mutated_campaign.budget.checked_sub(&old_remaining_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; + let new_remaining_for_mutated_campaign = mutated_campaign.budget.checked_sub(&spent_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; + sum_of_campaigns_remaining.checked_add(&new_remaining_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; Ok(sum_of_campaigns_remaining) } } @@ -224,6 +266,7 @@ mod test { use deadpool::managed::Object; use crate::{ db::redis_pool::{Manager, TESTS_POOL}, + campaign::update_campaign::CAMPAIGN_REMAINING_KEY, }; use std::convert::TryFrom; use super::*; @@ -260,10 +303,10 @@ mod test { } #[tokio::test] - async fn does_it_update_remaining() { + async fn does_it_increase_remaining() { let mut redis = get_redis().await; let campaign = get_campaign(); - let key = format!("remaining:{}", campaign.id); + let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, campaign.id); // Setting the redis base variable redis::cmd("SET") @@ -275,8 +318,8 @@ mod test { // 2 async calls at once, should be 500 after them futures::future::join( - update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)), - update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)) + increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)), + increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)) ).await; let remaining = redis::cmd("GET") @@ -290,7 +333,7 @@ mod test { let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); assert_eq!(remaining, UnifiedNum::from_u64(500)); - update_remaining_for_campaign(&redis, campaign.id, campaign.budget).await.expect("should increase"); + increase_remaining_for_campaign(&redis, campaign.id, campaign.budget).await.expect("should increase"); let remaining = redis::cmd("GET") .arg(&key) @@ -304,7 +347,7 @@ mod test { let should_be_remaining = UnifiedNum::from_u64(500).checked_add(&campaign.budget).expect("should add"); assert_eq!(remaining, should_be_remaining); - update_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(0)).await.expect("should work"); + increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(0)).await.expect("should work"); let remaining = redis::cmd("GET") .arg(&key) @@ -322,7 +365,7 @@ mod test { async fn update_remaining_before_it_is_set() { let mut redis = get_redis().await; let campaign = get_campaign(); - let key = format!("remaining:{}", campaign.id); + let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, campaign.id); let remaining = redis::cmd("GET") .arg(&key) @@ -337,8 +380,6 @@ mod test { // test get_remaining_for_multiple_campaigns // test get_remaining_for_campaign_from_redis - - // test get_spent_for_campaign } pub async fn insert_events( req: Request, diff --git a/sentry/src/routes/channel.rs b/sentry/src/routes/channel.rs index 304bd3432..ca337b6f7 100644 --- a/sentry/src/routes/channel.rs +++ b/sentry/src/routes/channel.rs @@ -3,7 +3,7 @@ use crate::db::{ get_channel_by_id, insert_channel, insert_validator_messages, list_channels, update_exhausted_channel, PoolError, }; -use crate::{success_response, Application, Auth, ResponseError, RouteParams, Session}; +use crate::{success_response, Application, Auth, ResponseError, RouteParams}; use futures::future::try_join_all; use hex::FromHex; use hyper::{Body, Request, Response}; @@ -11,7 +11,7 @@ use primitives::{ adapter::Adapter, sentry::{ channel_list::{ChannelListQuery, LastApprovedQuery}, - Event, LastApproved, LastApprovedResponse, SuccessResponse, + LastApproved, LastApprovedResponse, SuccessResponse, }, validator::MessageTypes, Channel, ChannelId, From 9fd4f4add7f77a49bb8915ab7c58609709e4827e Mon Sep 17 00:00:00 2001 From: simzzz Date: Wed, 30 Jun 2021 11:43:21 +0300 Subject: [PATCH 22/49] Included MGET for getting multiple remainings at once --- sentry/src/routes/campaign.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index f7e08f8e0..5895a7035 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -227,18 +227,20 @@ pub mod update_campaign { } async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign_id: CampaignId) -> Result, CampaignError> { - let other_campaigns_remaining = campaigns + let keys: Vec = campaigns.into_iter().map(|c| format!("{}:{}", *CAMPAIGN_REMAINING_KEY, c.id)).collect(); + let remainings = redis::cmd("MGET") + .arg(keys) + .query_async::<_, Vec>>(&mut redis.clone()) + .await?; + + let remainings = remainings .into_iter() - .filter(|c| c.id != mutated_campaign_id) - // TODO: Do 1 call with MGET - .map(|c| async move { - let remaining = get_remaining_for_campaign_from_redis(&redis, c.id).await.ok_or(CampaignError::FailedUpdate("Couldn't get remaining for campaign".to_string()))?; - Ok(remaining) - }) - .collect::>(); - let other_campaigns_remaining = join_all(other_campaigns_remaining).await; - let other_campaigns_remaining: Result, _> = other_campaigns_remaining.into_iter().collect(); - other_campaigns_remaining + .flat_map(|r| r) + .map(|r| get_unified_num_from_string(&r)) + .flatten() + .collect(); + + Ok(remainings) } pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: &Campaign) -> Result { From 93800894d62f5ec8b5ae4c24b68bbd13d64e25e7 Mon Sep 17 00:00:00 2001 From: simzzz Date: Wed, 30 Jun 2021 12:03:15 +0300 Subject: [PATCH 23/49] fixed failing test --- sentry/src/routes/campaign.rs | 152 +++++++++++++++++----------------- 1 file changed, 77 insertions(+), 75 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 5895a7035..5b174f380 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -258,6 +258,79 @@ pub mod update_campaign { } } +pub async fn insert_events( + req: Request, + app: &Application, +) -> Result, ResponseError> { + let (req_head, req_body) = req.into_parts(); + + let auth = req_head.extensions.get::(); + let session = req_head + .extensions + .get::() + .expect("request should have session"); + + let campaign = req_head + .extensions + .get::() + .expect("request should have a Campaign loaded"); + + let body_bytes = hyper::body::to_bytes(req_body).await?; + let mut request_body = serde_json::from_slice::>>(&body_bytes)?; + + let events = request_body + .remove("events") + .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; + + let processed = process_events(app, auth, session, campaign, events).await?; + + Ok(Response::builder() + .header("Content-type", "application/json") + .body(serde_json::to_string(&SuccessResponse { success: processed })?.into()) + .unwrap()) +} + +async fn process_events( + app: &Application, + auth: Option<&Auth>, + session: &Session, + campaign: &Campaign, + events: Vec, +) -> Result { + if &Utc::now() > &campaign.active.to { + return Err(ResponseError::BadRequest("Campaign is expired".into())); + } + + // + // TODO #381: AIP#61 Spender Aggregator should be called + // + + // handle events - check access + // handle events - Update targeting rules + // calculate payout + // distribute fees + // handle spending - Spender Aggregate + // handle events - aggregate Events and put into analytics + + check_access( + &app.redis, + session, + auth, + &app.config.ip_rate_limit, + &campaign, + &events, + ) + .await + .map_err(|e| match e { + access::Error::ForbiddenReferrer => ResponseError::Forbidden(e.to_string()), + access::Error::RulesError(error) => ResponseError::TooManyRequests(error), + access::Error::UnAuthenticated => ResponseError::Unauthorized, + _ => ResponseError::BadRequest(e.to_string()), + })?; + + Ok(true) +} + #[cfg(test)] mod test { use primitives::{ @@ -372,9 +445,10 @@ mod test { let remaining = redis::cmd("GET") .arg(&key) .query_async::<_, Option>(&mut redis.connection) - .await; + .await + .expect("should return None"); - assert_eq!(remaining.is_err(), true) + assert_eq!(remaining, None) } // test get_campaigns_remaining_sum @@ -382,76 +456,4 @@ mod test { // test get_remaining_for_multiple_campaigns // test get_remaining_for_campaign_from_redis -} -pub async fn insert_events( - req: Request, - app: &Application, -) -> Result, ResponseError> { - let (req_head, req_body) = req.into_parts(); - - let auth = req_head.extensions.get::(); - let session = req_head - .extensions - .get::() - .expect("request should have session"); - - let campaign = req_head - .extensions - .get::() - .expect("request should have a Campaign loaded"); - - let body_bytes = hyper::body::to_bytes(req_body).await?; - let mut request_body = serde_json::from_slice::>>(&body_bytes)?; - - let events = request_body - .remove("events") - .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; - - let processed = process_events(app, auth, session, campaign, events).await?; - - Ok(Response::builder() - .header("Content-type", "application/json") - .body(serde_json::to_string(&SuccessResponse { success: processed })?.into()) - .unwrap()) -} - -async fn process_events( - app: &Application, - auth: Option<&Auth>, - session: &Session, - campaign: &Campaign, - events: Vec, -) -> Result { - if &Utc::now() > &campaign.active.to { - return Err(ResponseError::BadRequest("Campaign is expired".into())); - } - - // - // TODO #381: AIP#61 Spender Aggregator should be called - // - - // handle events - check access - // handle events - Update targeting rules - // calculate payout - // distribute fees - // handle spending - Spender Aggregate - // handle events - aggregate Events and put into analytics - - check_access( - &app.redis, - session, - auth, - &app.config.ip_rate_limit, - &campaign, - &events, - ) - .await - .map_err(|e| match e { - access::Error::ForbiddenReferrer => ResponseError::Forbidden(e.to_string()), - access::Error::RulesError(error) => ResponseError::TooManyRequests(error), - access::Error::UnAuthenticated => ResponseError::Unauthorized, - _ => ResponseError::BadRequest(e.to_string()), - })?; - - Ok(true) -} +} \ No newline at end of file From 219e3f01dd3989ab04dd12fa82c70d81ca0ba0a5 Mon Sep 17 00:00:00 2001 From: simzzz Date: Wed, 30 Jun 2021 14:24:55 +0300 Subject: [PATCH 24/49] cargofmt + some refactoring + used MutatedCampaign object in more places --- sentry/src/db/campaign.rs | 7 +++---- sentry/src/lib.rs | 26 +++++++++++++++----------- sentry/src/routes/campaign.rs | 33 +++++++++++++++++---------------- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 4f4817e72..3b143cd0e 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -1,5 +1,5 @@ use crate::db::{DbPool, PoolError}; -use primitives::{ChannelId, CampaignId, Campaign}; +use primitives::{Campaign, CampaignId, ChannelId}; use tokio_postgres::types::Json; pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result { @@ -96,7 +96,7 @@ mod test { use crate::{ db::tests_postgres::{setup_test_migrations, DATABASE_POOL}, - ResponseError + ResponseError, }; use super::*; @@ -123,8 +123,7 @@ mod test { assert!(is_inserted); - let is_duplicate_inserted = insert_campaign(&database.pool, &campaign_for_testing) - .await; + let is_duplicate_inserted = insert_campaign(&database.pool, &campaign_for_testing).await; assert!(is_duplicate_inserted.is_err()); let insertion_error = is_duplicate_inserted.err().expect("should get error"); diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 842799c5d..69d203d7a 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::all)] #![deny(rust_2018_idioms)] -use crate::db::{DbPool, fetch_campaign}; +use crate::db::{fetch_campaign, DbPool}; use crate::routes::campaign; use crate::routes::channel::channel_status; use crate::routes::event_aggregate::list_channel_event_aggregates; @@ -18,7 +18,7 @@ use middleware::{campaign::CampaignLoad, Chain, Middleware}; use once_cell::sync::Lazy; use primitives::adapter::Adapter; use primitives::sentry::ValidationErrorResponse; -use primitives::{Config, ValidatorId, CampaignId}; +use primitives::{CampaignId, Config, ValidatorId}; use redis::aio::MultiplexedConnection; use regex::Regex; use routes::analytics::{advanced_analytics, advertiser_analytics, analytics, publisher_analytics}; @@ -201,14 +201,21 @@ async fn campaigns_router( .map_or("".to_string(), |m| m.as_str().to_string())]); // Should be safe to access indice here - let campaign_id = param.get(0).ok_or_else(|| ResponseError::BadRequest("No CampaignId".to_string()))?; - let campaign_id = CampaignId::from_str(&campaign_id).map_err(|_| ResponseError::BadRequest("Bad CampaignId".to_string()))?; + let campaign_id = param + .get(0) + .ok_or_else(|| ResponseError::BadRequest("No CampaignId".to_string()))?; + let campaign_id = CampaignId::from_str(&campaign_id) + .map_err(|_| ResponseError::BadRequest("Bad CampaignId".to_string()))?; - let campaign = fetch_campaign(app.pool.clone(), &campaign_id).await.map_err(|_| ResponseError::NotFound)?; + let campaign = fetch_campaign(app.pool.clone(), &campaign_id) + .await + .map_err(|_| ResponseError::NotFound)?; req.extensions_mut().insert(campaign); handle_route(req, app).await - } else if let (Some(caps), &Method::POST) = (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) { + } else if let (Some(caps), &Method::POST) = + (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) + { let param = RouteParams(vec![caps .get(1) .map_or("".to_string(), |m| m.as_str().to_string())]); @@ -245,8 +252,6 @@ async fn analytics_router( ) -> Result, ResponseError> { let (route, method) = (req.uri().path(), req.method()); - - // TODO AIP#61: Add routes for: // - POST /channel/:id/pay // #[serde(rename_all = "camelCase")] @@ -323,9 +328,8 @@ async fn channels_router( req.extensions_mut().insert(param); insert_events(req, app).await - } else */ - if let (Some(caps), &Method::GET) = (LAST_APPROVED_BY_CHANNEL_ID.captures(&path), method) - { + } else */ + if let (Some(caps), &Method::GET) = (LAST_APPROVED_BY_CHANNEL_ID.captures(&path), method) { let param = RouteParams(vec![caps .get(1) .map_or("".to_string(), |m| m.as_str().to_string())]); diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 5b174f380..79dd84bfb 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -21,17 +21,18 @@ use primitives::{ spender::Spendable, Campaign, CampaignId, UnifiedNum, BigNum }; -use redis::aio::MultiplexedConnection; +use redis::{ + aio::MultiplexedConnection, + RedisError, +}; use slog::error; use std::{ cmp::max, str::FromStr, + collections::HashMap, }; -use futures::future::join_all; -use std::collections::HashMap; use deadpool_postgres::PoolError; use tokio_postgres::error::SqlState; -use redis::RedisError; use chrono::Utc; @@ -147,8 +148,6 @@ pub mod update_campaign { let modified_campaign = ModifyCampaign::from_campaign(campaign.clone()); - let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); - // modify Campaign modify_campaign(&app.pool, &campaign, &modified_campaign, &app.redis).await.map_err(|_| ResponseError::BadRequest("Failed to update campaign".to_string()))?; @@ -159,20 +158,22 @@ pub mod update_campaign { // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 + + let new_budget = modified_campaign.budget.ok_or(CampaignError::FailedUpdate("Couldn't get new budget".to_string()))?; let accounting_spent = get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; let old_remaining = get_remaining_for_campaign_from_redis(&redis, campaign.id).await.ok_or(CampaignError::FailedUpdate("Couldn't get remaining for campaign".to_string()))?; - let campaign_spent = campaign.budget.checked_sub(&old_remaining).ok_or(CampaignError::CalculationError)?; - if campaign_spent >= campaign.budget { + let campaign_spent = new_budget.checked_sub(&old_remaining).ok_or(CampaignError::CalculationError)?; + if campaign_spent >= new_budget { return Err(CampaignError::BudgetExceeded); } let old_remaining = UnifiedNum::from_u64(max(0, old_remaining.to_u64())); - let new_remaining = campaign.budget.checked_sub(&campaign_spent).ok_or(CampaignError::CalculationError)?; + let new_remaining = new_budget.checked_sub(&campaign_spent).ok_or(CampaignError::CalculationError)?; if new_remaining >= old_remaining { let diff_in_remaining = new_remaining.checked_sub(&old_remaining).ok_or(CampaignError::CalculationError)?; @@ -186,7 +187,7 @@ pub mod update_campaign { // Gets the latest Spendable for this (spender, channelId) pair let total_remaining = get_total_remaining_for_channel(&accounting_spent, &latest_spendable).ok_or(CampaignError::FailedUpdate("Could not get total remaining for channel".to_string()))?; let campaigns_for_channel = get_campaigns_by_channel(&pool, &campaign.channel.id()).await?; - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, &campaign).await.map_err(|_| CampaignError::CalculationError)?; + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, campaign.id, &new_budget).await.map_err(|_| CampaignError::CalculationError)?; if campaigns_remaining_sum > total_remaining { return Err(CampaignError::BudgetExceeded); } @@ -226,7 +227,7 @@ pub mod update_campaign { remaining } - async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign_id: CampaignId) -> Result, CampaignError> { + async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign]) -> Result, CampaignError> { let keys: Vec = campaigns.into_iter().map(|c| format!("{}:{}", *CAMPAIGN_REMAINING_KEY, c.id)).collect(); let remainings = redis::cmd("MGET") .arg(keys) @@ -243,16 +244,16 @@ pub mod update_campaign { Ok(remainings) } - pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: &Campaign) -> Result { - let other_campaigns_remaining = get_remaining_for_multiple_campaigns(&redis, &campaigns, mutated_campaign.id).await?; + pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: CampaignId, new_budget: &UnifiedNum) -> Result { + let other_campaigns_remaining = get_remaining_for_multiple_campaigns(&redis, &campaigns).await?; let sum_of_campaigns_remaining = other_campaigns_remaining .into_iter() .try_fold(UnifiedNum::from_u64(0), |sum, val| sum.checked_add(&val).ok_or(CampaignError::CalculationError))?; // Necessary to do it explicitly for current campaign as its budget is not yet updated in DB - let old_remaining_for_mutated_campaign = get_remaining_for_campaign_from_redis(&redis, mutated_campaign.id).await.ok_or(CampaignError::CalculationError)?; - let spent_for_mutated_campaign = mutated_campaign.budget.checked_sub(&old_remaining_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; - let new_remaining_for_mutated_campaign = mutated_campaign.budget.checked_sub(&spent_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; + let old_remaining_for_mutated_campaign = get_remaining_for_campaign_from_redis(&redis, mutated_campaign).await.ok_or(CampaignError::CalculationError)?; + let spent_for_mutated_campaign = new_budget.checked_sub(&old_remaining_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; + let new_remaining_for_mutated_campaign = new_budget.checked_sub(&spent_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; sum_of_campaigns_remaining.checked_add(&new_remaining_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; Ok(sum_of_campaigns_remaining) } From ccb1eac52f289a2bca604daee013a0120a5da9bf Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 30 Jun 2021 15:08:50 +0300 Subject: [PATCH 25/49] sentry - Cargo - add `postgres-types` `derive` feature --- Cargo.lock | 13 +++++++++++++ sentry/Cargo.toml | 1 + 2 files changed, 14 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 57dbdc1fe..f27f5d2a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2729,6 +2729,17 @@ dependencies = [ "tokio-postgres", ] +[[package]] +name = "postgres-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c857dd221cb0e7d8414b894a0ce29eae44d453dda0baa132447878e75e701477" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "postgres-native-tls" version = "0.5.0" @@ -2769,6 +2780,7 @@ dependencies = [ "bytes 1.0.1", "chrono", "fallible-iterator", + "postgres-derive", "postgres-protocol", "serde", "serde_json", @@ -3496,6 +3508,7 @@ dependencies = [ "lazy_static", "migrant_lib", "once_cell", + "postgres-types", "primitives", "redis", "regex", diff --git a/sentry/Cargo.toml b/sentry/Cargo.toml index 7e25ca25c..746ee77da 100644 --- a/sentry/Cargo.toml +++ b/sentry/Cargo.toml @@ -29,6 +29,7 @@ redis = { version = "0.19", features = ["aio", "tokio-comp"] } deadpool = "0.8.0" deadpool-postgres = "0.8.0" tokio-postgres = { version = "0.7.0", features = ["with-chrono-0_4", "with-serde_json-1"] } +postgres-types = { version = "0.2.1", features = ["derive", "with-chrono-0_4", "with-serde_json-1"] } # Migrations migrant_lib = { version = "^0.32", features = ["d-postgres"] } From 99807db8fbe6b036355f9a417612761aa133fb16 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 30 Jun 2021 15:09:06 +0300 Subject: [PATCH 26/49] sentry - migrations - accounting - flat structure --- .../migrations/20190806011140_initial-tables/up.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sentry/migrations/20190806011140_initial-tables/up.sql b/sentry/migrations/20190806011140_initial-tables/up.sql index 710e44898..f611fafd0 100644 --- a/sentry/migrations/20190806011140_initial-tables/up.sql +++ b/sentry/migrations/20190806011140_initial-tables/up.sql @@ -68,13 +68,16 @@ CREATE AGGREGATE jsonb_object_agg (jsonb) ( INITCOND = '{}' ); +CREATE TYPE AccountingSide AS ENUM ('Earner', 'Spender'); + CREATE TABLE accounting ( channel_id varchar(66) NOT NULL, - channel jsonb NOT NULL, - earners jsonb DEFAULT '{}' NULL, - spenders jsonb DEFAULT '{}' NULL, + side AccountingSide NOT NULL, + "address" varchar(42) NOT NULL, + amount bigint NOT NULL, updated timestamp(2) with time zone DEFAULT NULL NULL, created timestamp(2) with time zone NOT NULL, - PRIMARY KEY (channel_id) -) \ No newline at end of file + -- Do not rename the Primary key constraint (`accounting_pkey`)! + PRIMARY KEY (channel_id, side, "address") +); \ No newline at end of file From a3eb1017a7e10505e98aa25d7f050840a13aa74e Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 30 Jun 2021 15:15:30 +0300 Subject: [PATCH 27/49] sentry - db - accounting follows flat struture --- primitives/src/sentry/accounting.rs | 45 +--- sentry/src/db/accounting.rs | 319 +++++++++++++++++++++++----- 2 files changed, 280 insertions(+), 84 deletions(-) diff --git a/primitives/src/sentry/accounting.rs b/primitives/src/sentry/accounting.rs index 6c5b1b88d..64589ce23 100644 --- a/primitives/src/sentry/accounting.rs +++ b/primitives/src/sentry/accounting.rs @@ -1,9 +1,6 @@ -use std::{ - convert::TryFrom, - marker::PhantomData, -}; +use std::{convert::TryFrom, marker::PhantomData}; -use crate::{balances_map::UnifiedMap, Address, channel_v5::Channel, UnifiedNum}; +use crate::{balances_map::UnifiedMap, channel_v5::Channel, Address, UnifiedNum}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Deserializer, Serialize}; use thiserror::Error; @@ -73,9 +70,11 @@ impl Balances { } } -#[derive(Debug)] +#[derive(Debug, Error)] pub enum OverflowError { + #[error("Spender {0} amount overflowed")] Spender(Address), + #[error("Earner {0} amount overflowed")] Earner(Address), } @@ -130,7 +129,8 @@ mod de { Ok(Self { channel: de_acc.channel, - balances: Balances::::try_from(de_acc.balances).map_err(serde::de::Error::custom)?, + balances: Balances::::try_from(de_acc.balances) + .map_err(serde::de::Error::custom)?, created: de_acc.created, updated: de_acc.updated, }) @@ -146,7 +146,10 @@ mod de { Ok(Self { channel: unchecked_acc.channel, - balances: unchecked_acc.balances.check().map_err(serde::de::Error::custom)?, + balances: unchecked_acc + .balances + .check() + .map_err(serde::de::Error::custom)?, created: unchecked_acc.created, updated: unchecked_acc.updated, }) @@ -195,29 +198,3 @@ mod de { } } } - -#[cfg(feature = "postgres")] -mod postgres { - use super::*; - use postgres_types::Json; - use tokio_postgres::Row; - - impl TryFrom<&Row> for Accounting { - type Error = Error; - - fn try_from(row: &Row) -> Result { - let balances = Balances:: { - earners: row.get::<_, Json<_>>("earners").0, - spenders: row.get::<_, Json<_>>("spenders").0, - state: PhantomData::default(), - }.check()?; - - Ok(Self { - channel: row.get("channel"), - balances, - updated: row.get("updated"), - created: row.get("created"), - }) - } - } -} \ No newline at end of file diff --git a/sentry/src/db/accounting.rs b/sentry/src/db/accounting.rs index ffc8e3db4..fe1f0b0d4 100644 --- a/sentry/src/db/accounting.rs +++ b/sentry/src/db/accounting.rs @@ -1,12 +1,8 @@ -use std::convert::TryFrom; - use chrono::{DateTime, Utc}; use primitives::{ - channel_v5::Channel, - sentry::accounting::{Accounting, Balances, CheckedState}, Address, ChannelId, UnifiedNum, }; -use tokio_postgres::types::Json; +use tokio_postgres::{IsolationLevel, Row, types::{FromSql, ToSql}}; use super::{DbPool, PoolError}; use thiserror::Error; @@ -19,49 +15,117 @@ pub enum Error { Postgres(#[from] PoolError), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Accounting { + pub channel_id: ChannelId, + pub side: Side, + pub address: Address, + pub amount: UnifiedNum, + pub updated: Option>, + pub created: DateTime, +} + +impl From<&Row> for Accounting { + fn from(row: &Row) -> Self { + Self { + channel_id: row.get("channel_id"), + side: row.get("side"), + address: row.get("address"), + amount: row.get("amount"), + updated: row.get("updated"), + created: row.get("created"), + } + } +} + +#[derive(Debug, Clone, Copy, ToSql, FromSql, PartialEq, Eq)] +#[postgres(name = "accountingside")] +pub enum Side { + Earner, + Spender, +} + +pub enum SpendError { + Pool(PoolError), + NoRecordsUpdated, +} + /// ```text -/// SELECT (spenders ->> $1)::bigint as spent FROM accounting WHERE channel_id = $2 +/// SELECT channel_id, side, address, amount, updated, created FROM accounting WHERE channel_id = $1 AND address = $2 AND side = $3 /// ``` -/// This function returns the spent amount in a `Channel` of a given spender -pub async fn get_accounting_spent( +pub async fn get_accounting( pool: DbPool, - spender: &Address, - channel_id: &ChannelId, -) -> Result { + channel_id: ChannelId, + address: Address, + side: Side, +) -> Result, PoolError> { let client = pool.get().await?; let statement = client - .prepare("SELECT (spenders ->> $1)::bigint as spent FROM accounting WHERE channel_id = $2") + .prepare("SELECT channel_id, side, address, amount, updated, created FROM accounting WHERE channel_id = $1 AND address = $2 AND side = $3") .await?; - let row = client.query_one(&statement, &[spender, channel_id]).await?; + let row = client + .query_opt(&statement, &[&channel_id, &address, &side]) + .await?; - Ok(row.get("spent")) + Ok(row.as_ref().map(Accounting::from)) } -// TODO This is still WIP -#[allow(dead_code)] -async fn insert_accounting( +/// Will update current Spender/Earner amount or insert a new Accounting record +/// +/// See `UPDATE_ACCOUNTING_STATEMENT` static for full query. +static UPDATE_ACCOUNTING_STATEMENT: &str = "INSERT INTO accounting(channel_id, side, address, amount, updated, created) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT ON CONSTRAINT accounting_pkey DO UPDATE SET amount = accounting.amount + $4, updated = $6 WHERE accounting.channel_id = $1 AND accounting.side = $2 AND accounting.address = $3 RETURNING channel_id, side, address, amount, updated, created"; +pub async fn update_accounting( pool: DbPool, - channel: Channel, - balances: Balances, -) -> Result, Error> { + channel_id: ChannelId, + address: Address, + side: Side, + amount: UnifiedNum, +) -> Result { let client = pool.get().await?; + let statement = client + .prepare(UPDATE_ACCOUNTING_STATEMENT) + .await?; - let statement = client.prepare("INSERT INTO accounting (channel_id, channel, earners, spenders, updated, created) VALUES ($1, $2, $3, $4, $5, NOW()) RETURNING channel, earners, spenders, updated, created").await.map_err(PoolError::Backend)?; - - let earners = Json(&balances.earners); - let spenders = Json(&balances.spenders); + let now = Utc::now(); let updated: Option> = None; let row = client .query_one( &statement, - &[&channel.id(), &channel, &earners, &spenders, &updated], + &[&channel_id, &side, &address, &amount, &updated, &now], ) - .await - .map_err(PoolError::Backend)?; + .await?; + + Ok(Accounting::from(&row)) +} + +/// Will use `UPDATE_ACCOUNTING_STATEMENT` to create and run the query twice - once for Earner and once for Spender accounting. +/// +/// It runs both queries in a transaction in order to rollback if one of the queries fails. +pub async fn spend_accountings( + pool: DbPool, + channel_id: ChannelId, + earner: Address, + spender: Address, + amount: UnifiedNum, +) -> Result<(Accounting, Accounting), PoolError> { + let mut client = pool.get().await?; + + // The reads and writes in this transaction must be able to be committed as an atomic “unit” with respect to reads and writes of all other concurrent serializable transactions without interleaving. + let transaction = client.build_transaction().isolation_level(IsolationLevel::Serializable).start().await?; + + let statement = transaction.prepare(UPDATE_ACCOUNTING_STATEMENT).await?; + + let now = Utc::now(); + let updated: Option> = None; + + let earner_row = transaction.query_one(&statement, &[&channel_id, &Side::Earner, &earner, &amount, &updated, &now]).await?; + let spender_row = transaction.query_one(&statement, &[&channel_id, &Side::Spender, &spender, &amount, &updated, &now]).await?; + + transaction.commit().await?; - Accounting::try_from(&row).map_err(Error::Balances) + Ok((Accounting::from(&earner_row), Accounting::from(&spender_row))) } #[cfg(test)] @@ -73,40 +137,195 @@ mod test { use super::*; #[tokio::test] - async fn get_spent() { + async fn insert_update_and_get_accounting() { let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); setup_test_migrations(database.pool.clone()) .await .expect("Migrations should succeed"); - let channel = DUMMY_CAMPAIGN.channel.clone(); - - let spender = ADDRESSES["creator"]; + let channel_id = DUMMY_CAMPAIGN.channel.id(); let earner = ADDRESSES["publisher"]; + let spender = ADDRESSES["creator"]; + + let amount = UnifiedNum::from(100_000_000); + let update_amount = UnifiedNum::from(200_000_000); - let mut balances = Balances::default(); - let spend_amount = UnifiedNum::from(100); - balances - .spend(spender, earner, spend_amount) - .expect("Should be ok"); + // Spender insert/update + { + let inserted = update_accounting( + database.pool.clone(), + channel_id, + spender, + Side::Spender, + amount, + ) + .await + .expect("Should insert"); + assert_eq!(spender, inserted.address); + assert_eq!(Side::Spender, inserted.side); + assert_eq!(UnifiedNum::from(100_000_000), inserted.amount); - let accounting = insert_accounting(database.pool.clone(), channel.clone(), balances) + let updated = update_accounting( + database.pool.clone(), + channel_id, + spender, + Side::Spender, + update_amount, + ) .await .expect("Should insert"); + assert_eq!(spender, updated.address); + assert_eq!(Side::Spender, updated.side); + assert_eq!( + UnifiedNum::from(300_000_000), + updated.amount, + "Should add the newly spent amount to the existing one" + ); + + let spent = get_accounting(database.pool.clone(), channel_id, spender, Side::Spender).await.expect("Should query for the updated accounting"); + assert_eq!(Some(updated), spent); + + let earned = get_accounting(database.pool.clone(), channel_id, spender, Side::Earner).await.expect("Should query for accounting"); + assert!(earned.is_none(), "Spender shouldn't have an earned amount"); + } - let spent = get_accounting_spent(database.pool.clone(), &spender, &channel.id()) + // Earner insert/update + { + let inserted = update_accounting( + database.pool.clone(), + channel_id, + earner, + Side::Earner, + amount, + ) .await - .expect("Should get spent"); - - assert_eq!(spend_amount, spent); - assert_eq!( - accounting - .balances - .spenders - .get(&spender) - .expect("Should contain value"), - &spent - ); + .expect("Should insert"); + assert_eq!(earner, inserted.address); + assert_eq!(Side::Earner, inserted.side); + assert_eq!(UnifiedNum::from(100_000_000), inserted.amount); + + let updated = update_accounting( + database.pool.clone(), + channel_id, + earner, + Side::Earner, + update_amount, + ) + .await + .expect("Should insert"); + assert_eq!(earner, updated.address); + assert_eq!(Side::Earner, updated.side); + assert_eq!( + UnifiedNum::from(300_000_000), + updated.amount, + "Should add the newly earned amount to the existing one" + ); + + let earned = get_accounting(database.pool.clone(), channel_id, earner, Side::Earner).await.expect("Should query for the updated accounting"); + assert_eq!(Some(updated), earned); + + let spent = get_accounting(database.pool.clone(), channel_id, earner, Side::Spender).await.expect("Should query for accounting"); + assert!(spent.is_none(), "Earner shouldn't have a spent amount"); + } + + + // Spender as Earner & another Spender + // Will test the previously spent amount as well! + { + let spender_as_earner = spender; + + let inserted = update_accounting( + database.pool.clone(), + channel_id, + spender_as_earner, + Side::Earner, + amount, + ) + .await + .expect("Should insert"); + assert_eq!(spender_as_earner, inserted.address); + assert_eq!(Side::Earner, inserted.side); + assert_eq!(UnifiedNum::from(100_000_000), inserted.amount); + + let updated = update_accounting( + database.pool.clone(), + channel_id, + spender_as_earner, + Side::Earner, + UnifiedNum::from(999), + ) + .await + .expect("Should insert"); + assert_eq!(spender, updated.address); + assert_eq!(Side::Earner, updated.side); + assert_eq!( + UnifiedNum::from(100_000_999), + updated.amount, + "Should add the newly spent amount to the existing one" + ); + + let earned_acc = get_accounting(database.pool.clone(), channel_id, spender_as_earner, Side::Earner).await.expect("Should query for earned accounting").expect("Should have Earned accounting for Spender as Earner"); + assert_eq!(UnifiedNum::from(100_000_999), earned_acc.amount); + + let spent_acc = get_accounting(database.pool.clone(), channel_id, spender_as_earner, Side::Spender).await.expect("Should query for spent accounting").expect("Should have Spent accounting for Spender as Earner"); + assert_eq!(UnifiedNum::from(300_000_000), spent_acc.amount); + + } + } + + #[tokio::test] + async fn test_spending_accountings() { + let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); + + setup_test_migrations(database.pool.clone()) + .await + .expect("Migrations should succeed"); + + let channel_id = DUMMY_CAMPAIGN.channel.id(); + let earner = ADDRESSES["publisher"]; + let spender = ADDRESSES["creator"]; + let other_spender = ADDRESSES["tester"]; + + let amount = UnifiedNum::from(100_000_000); + let update_amount = UnifiedNum::from(200_000_000); + + // Spender & Earner insert + let (inserted_earner, inserted_spender) = spend_accountings(database.pool.clone(), channel_id, earner, spender, amount).await.expect("Should insert Earner and Spender"); + assert_eq!(earner, inserted_earner.address); + assert_eq!(Side::Earner, inserted_earner.side); + assert_eq!(UnifiedNum::from(100_000_000), inserted_earner.amount); + + assert_eq!(spender, inserted_spender.address); + assert_eq!(Side::Spender, inserted_spender.side); + assert_eq!(UnifiedNum::from(100_000_000), inserted_spender.amount); + + // Spender & Earner update + let (updated_earner, updated_spender) = spend_accountings(database.pool.clone(), channel_id, earner, spender, update_amount).await.expect("Should update Earner and Spender"); + + assert_eq!(earner, updated_earner.address); + assert_eq!(Side::Earner, updated_earner.side); + assert_eq!(UnifiedNum::from(300_000_000), updated_earner.amount, "Should add the newly earned amount to the existing one"); + + assert_eq!(spender, updated_spender.address); + assert_eq!(Side::Spender, updated_spender.side); + assert_eq!(UnifiedNum::from(300_000_000), updated_spender.amount, "Should add the newly spend amount to the existing one"); + + // Spender as an Earner & another spender + let (spender_as_earner, inserted_other_spender) = spend_accountings(database.pool.clone(), channel_id, spender, other_spender, UnifiedNum::from(999)).await.expect("Should update Spender as Earner and the Other Spender"); + + assert_eq!(spender, spender_as_earner.address); + assert_eq!(Side::Earner, spender_as_earner.side); + assert_eq!(UnifiedNum::from(999), spender_as_earner.amount, "Should add earner accounting for the previous Spender"); + + assert_eq!(other_spender, inserted_other_spender.address); + assert_eq!(Side::Spender, inserted_other_spender.side); + assert_eq!(UnifiedNum::from(999), inserted_other_spender.amount); + + let earned = get_accounting(database.pool.clone(), channel_id, spender, Side::Earner).await.expect("Should query for accounting").expect("Should have Earned accounting for Spender as Earner"); + assert_eq!(UnifiedNum::from(999), earned.amount); + + let spent = get_accounting(database.pool.clone(), channel_id, spender, Side::Spender).await.expect("Should query for accounting").expect("Should have Spent accounting for Spender as Earner"); + assert_eq!(UnifiedNum::from(300_000_000), spent.amount); } } From ad2d585eec05dc6c37ed3a20ec16db005e6d89c3 Mon Sep 17 00:00:00 2001 From: simzzz Date: Tue, 6 Jul 2021 18:36:21 +0300 Subject: [PATCH 28/49] Finished with PR changes requested --- sentry/src/db/campaign.rs | 118 +++++++++++--- sentry/src/lib.rs | 10 +- sentry/src/routes/campaign.rs | 297 ++++++++++++++++------------------ 3 files changed, 239 insertions(+), 186 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 3b143cd0e..28b04a41e 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -1,12 +1,12 @@ use crate::db::{DbPool, PoolError}; -use primitives::{Campaign, CampaignId, ChannelId}; +use primitives::{sentry::campaign_create::ModifyCampaign, Campaign, CampaignId, ChannelId}; use tokio_postgres::types::Json; pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let ad_units = Json(campaign.ad_units.clone()); let stmt = client.prepare("INSERT INTO campaigns (id, channel_id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)").await?; - let row = client + let inserted = client .execute( &stmt, &[ @@ -28,7 +28,7 @@ pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result Result { +pub async fn update_campaign( + pool: &DbPool, + campaign: &Campaign, + modified_campaign: &ModifyCampaign, +) -> Result { let client = pool.get().await?; let statement = client .prepare("UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 WHERE id = $8") .await?; - let row = client + let ad_units = &modified_campaign + .ad_units + .as_ref() + .unwrap_or(&campaign.ad_units); + let ad_units = Json(ad_units); + + let updated_rows = client .execute( &statement, &[ - &campaign.budget, - &campaign.validators, - &campaign.title, - &campaign.pricing_bounds, - &campaign.event_submission, - &campaign.ad_units, - &campaign.targeting_rules, + &modified_campaign.budget.unwrap_or(campaign.budget), + &modified_campaign + .validators + .as_ref() + .unwrap_or(&campaign.validators), + &modified_campaign + .title + .as_ref() + .ok_or_else(|| &campaign.title) + .unwrap(), + &modified_campaign + .pricing_bounds + .as_ref() + .ok_or_else(|| &campaign.title) + .unwrap(), + &modified_campaign + .event_submission + .as_ref() + .ok_or_else(|| &campaign.title) + .unwrap(), + &ad_units, + &modified_campaign + .targeting_rules + .as_ref() + .unwrap_or(&campaign.targeting_rules), &campaign.id, ], ) .await?; - let exists = row == 1; + let exists = updated_rows == 1; Ok(exists) } #[cfg(test)] mod test { - use primitives::util::tests::prep_db::DUMMY_CAMPAIGN; + use primitives::{ + util::tests::prep_db::{DUMMY_CAMPAIGN, DUMMY_AD_UNITS}, + event_submission::{Rule, RateLimit}, + targeting::Rules, + UnifiedNum, EventSubmission, + }; + use primitives::campaign; + use std::time::Duration; use tokio_postgres::error::SqlState; use crate::{ @@ -125,14 +159,10 @@ mod test { let is_duplicate_inserted = insert_campaign(&database.pool, &campaign_for_testing).await; - assert!(is_duplicate_inserted.is_err()); - let insertion_error = is_duplicate_inserted.err().expect("should get error"); - match insertion_error { - PoolError::Backend(error) if error.code() == Some(&SqlState::UNIQUE_VIOLATION) => { - assert!(true); - } - _ => assert!(false), - } + assert!(matches!( + is_duplicate_inserted, + Err(PoolError::Backend(error)) if error.code() == Some(&SqlState::UNIQUE_VIOLATION) + )); let fetched_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing.id) .await @@ -140,4 +170,46 @@ mod test { assert_eq!(Some(campaign_for_testing), fetched_campaign); } + + #[tokio::test] + async fn it_updates_campaign() { + let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); + + setup_test_migrations(database.pool.clone()) + .await + .expect("Migrations should succeed"); + + let campaign_for_testing = DUMMY_CAMPAIGN.clone(); + + let is_inserted = insert_campaign(&database.pool, &campaign_for_testing) + .await + .expect("Should succeed"); + + assert!(is_inserted); + + let rule = Rule { + uids: None, + rate_limit: Some(RateLimit { + limit_type: "sid".to_string(), + time_frame: Duration::from_millis(20_000), + }), + }; + let new_budget = campaign_for_testing.budget + UnifiedNum::from_u64(1_000_000_000); + let modified_campaign = ModifyCampaign { + // pub budget: Option, + budget: Some(new_budget), + validators: None, + title: Some("Modified Campaign".to_string()), + pricing_bounds: Some(campaign::PricingBounds { + impression: Some(campaign::Pricing { min: 1.into(), max: 10.into()}), + click: Some(campaign::Pricing { min: 0.into(), max: 0.into()}) + }), + event_submission: Some(EventSubmission { allow: vec![rule] }), + ad_units: Some(DUMMY_AD_UNITS.to_vec()), + targeting_rules: Some(Rules::new()), + }; + + let is_campaign_updated = update_campaign(&database.pool, &campaign_for_testing, &modified_campaign).await.expect("should update"); + assert!(is_campaign_updated); + } } diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 69d203d7a..f98886b5a 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -22,7 +22,7 @@ use primitives::{CampaignId, Config, ValidatorId}; use redis::aio::MultiplexedConnection; use regex::Regex; use routes::analytics::{advanced_analytics, advertiser_analytics, analytics, publisher_analytics}; -use routes::campaign::{create_campaign, update_campaign::handle_route}; +use routes::campaign::{create_campaign, update_campaign}; use routes::cfg::config; use routes::channel::{ channel_list, channel_validate, create_channel, create_validator_messages, last_approved, @@ -63,8 +63,6 @@ lazy_static! { static ref ADVERTISER_ANALYTICS_BY_CHANNEL_ID: Regex = Regex::new(r"^/analytics/for-advertiser/0x([a-zA-Z0-9]{64})/?$").expect("The regex should be valid"); static ref PUBLISHER_ANALYTICS_BY_CHANNEL_ID: Regex = Regex::new(r"^/analytics/for-publisher/0x([a-zA-Z0-9]{64})/?$").expect("The regex should be valid"); static ref CREATE_EVENTS_BY_CHANNEL_ID: Regex = Regex::new(r"^/channel/0x([a-zA-Z0-9]{64})/events/?$").expect("The regex should be valid"); - static ref CAMPAIGN_UPDATE_BY_ID: Regex = - Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/?$").expect("The regex should be valid"); } static INSERT_EVENTS_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { @@ -73,6 +71,9 @@ static INSERT_EVENTS_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { static CLOSE_CAMPAIGN_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/close/?$").expect("The regex should be valid") }); +static CAMPAIGN_UPDATE_BY_ID: Lazy = Lazy::new(|| { + Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/?$").expect("The regex should be valid") +}); #[derive(Debug, Clone)] pub struct RouteParams(pub Vec); @@ -194,7 +195,6 @@ async fn campaigns_router( ) -> Result, ResponseError> { let (path, method) = (req.uri().path(), req.method()); - // create events if let (Some(caps), &Method::POST) = (CAMPAIGN_UPDATE_BY_ID.captures(&path), method) { let param = RouteParams(vec![caps .get(1) @@ -212,7 +212,7 @@ async fn campaigns_router( .map_err(|_| ResponseError::NotFound)?; req.extensions_mut().insert(campaign); - handle_route(req, app).await + update_campaign::handle_route(req, app).await } else if let (Some(caps), &Method::POST) = (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) { diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 79dd84bfb..8b0e5867a 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -6,7 +6,7 @@ use crate::{ campaign::{update_campaign, insert_campaign, get_campaigns_by_channel}, DbPool }, - routes::campaign::update_campaign::increase_remaining_for_campaign, + routes::campaign::update_campaign::set_initial_remaining_for_campaign, access::{self, check_access}, Auth, Session }; @@ -18,8 +18,8 @@ use primitives::{ campaign_create::{CreateCampaign, ModifyCampaign}, Event, SuccessResponse, }, - spender::Spendable, - Campaign, CampaignId, UnifiedNum, BigNum + campaign_validator::Validator, + Address, Campaign, CampaignId, UnifiedNum }; use redis::{ aio::MultiplexedConnection, @@ -27,38 +27,42 @@ use redis::{ }; use slog::error; use std::{ - cmp::max, - str::FromStr, + cmp::{max, Ordering}, collections::HashMap, + convert::TryInto, }; use deadpool_postgres::PoolError; use tokio_postgres::error::SqlState; use chrono::Utc; +use thiserror::Error; -#[derive(Debug, PartialEq, Eq)] -pub enum CampaignError { +#[derive(Debug, Error)] +pub enum Error { + #[error("Error while updating campaign: {0}")] FailedUpdate(String), - CalculationError, + #[error("Error while performing calculations")] + Calculation, + #[error("Error: Budget has been exceeded")] BudgetExceeded, -} - -impl From for CampaignError { - fn from(err: RedisError) -> Self { - CampaignError::FailedUpdate(err.to_string()) - } -} - -impl From for CampaignError { - fn from(err: PoolError) -> Self { - CampaignError::FailedUpdate(err.to_string()) - } + #[error("Error with new budget: {0}")] + NewBudget(String), + #[error("Redis error: {0}")] + Redis(#[from] RedisError), + #[error("DB Pool error: {0}")] + Pool(#[from] PoolError) } pub async fn create_campaign( req: Request, app: &Application, ) -> Result, ResponseError> { + let session = req + .extensions() + .get::() + .expect("request should have session") + .to_owned(); + let body = hyper::body::to_bytes(req.into_body()).await?; let campaign = serde_json::from_slice::(&body) @@ -66,7 +70,12 @@ pub async fn create_campaign( // create the actual `Campaign` with random `CampaignId` .into_campaign(); - // TODO AIP#61: Validate Campaign + campaign.validate(&app.config, &app.adapter.whoami()).map_err(|_| ResponseError::FailedValidation("couldn't valdiate campaign".to_string()))?; + + // TODO: Just use session.uid once it's address + if Address::from_bytes(session.uid.as_bytes()) != campaign.creator { + return Err(ResponseError::Forbidden("Request not sent by campaign creator".to_string())) + } let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); @@ -74,14 +83,16 @@ pub async fn create_campaign( // TODO: AIP#61: Update when changes to Spendable are ready let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - let remaining_for_channel = get_total_remaining_for_channel(&accounting_spent, &latest_spendable).ok_or(ResponseError::BadRequest("couldn't get total remaining for channel".to_string()))?; + let total_deposited = latest_spendable.deposit.total; + + let remaining_for_channel = total_deposited.checked_sub(&accounting_spent).ok_or(ResponseError::FailedValidation("couldn't calculate remaining for channel".to_string()))?; if campaign.budget > remaining_for_channel { return Err(ResponseError::BadRequest("Not Enough budget for campaign".to_string())); } - // If the channel is being created, the amount spent is 0, therefore remaining = budget - increase_remaining_for_campaign(&app.redis.clone(), campaign.id, campaign.budget).await.map_err(|_| ResponseError::BadRequest("Couldn't update remaining while creating campaign".to_string()))?; + // If the campaign is being created, the amount spent is 0, therefore remaining = budget + set_initial_remaining_for_campaign(&app.redis.clone(), campaign.id, campaign.budget).await.map_err(|_| ResponseError::BadRequest("Couldn't update remaining while creating campaign".to_string()))?; // insert Campaign match insert_campaign(&app.pool, &campaign).await { @@ -103,26 +114,15 @@ pub async fn create_campaign( Ok(success_response(serde_json::to_string(&campaign)?)) } - -// tested -fn get_total_remaining_for_channel(accounting_spent: &UnifiedNum, latest_spendable: &Spendable) -> Option { - let total_deposited = latest_spendable.deposit.total; - - let total_remaining = total_deposited.checked_sub(&accounting_spent); - total_remaining -} - pub mod update_campaign { use super::*; - use lazy_static::lazy_static; + use crate::fetch_campaign; - lazy_static!{ - pub static ref CAMPAIGN_REMAINING_KEY: &'static str = "campaignRemaining"; - } + pub const CAMPAIGN_REMAINING_KEY: &'static str = "campaignRemaining"; - pub async fn increase_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { - let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, id); - redis::cmd("INCRBY") + pub async fn set_initial_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); + redis::cmd("SETNX") .arg(&key) .arg(amount.to_u64()) .query_async(&mut redis.clone()) @@ -130,9 +130,9 @@ pub mod update_campaign { Ok(true) } - pub async fn decrease_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { - let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, id); - redis::cmd("DECRBY") + pub async fn increase_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); + redis::cmd("INCRBY") .arg(&key) .arg(amount.to_u64()) .query_async(&mut redis.clone()) @@ -140,121 +140,143 @@ pub mod update_campaign { Ok(true) } + pub async fn decrease_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Option { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); + let value = match redis::cmd("DECRBY") + .arg(&key) + .arg(amount.to_u64()) + .query_async::<_, Option>(&mut redis.clone()) + .await + { + Ok(Some(remaining)) => { + // Can't be less than 0 due to max() + Some(UnifiedNum::from_u64(max(0, remaining).try_into().unwrap())) + }, + _ => None + }; + value + } + pub async fn handle_route( req: Request, app: &Application, ) -> Result, ResponseError> { let campaign = req.extensions().get::().expect("We must have a campaign in extensions"); - let modified_campaign = ModifyCampaign::from_campaign(campaign.clone()); + // Getting the campaign as it was before the update operation, should exist + let campaign_being_mutated = fetch_campaign(app.pool.clone(), &campaign.id).await?.ok_or(ResponseError::NotFound)?; + // modify Campaign - modify_campaign(&app.pool, &campaign, &modified_campaign, &app.redis).await.map_err(|_| ResponseError::BadRequest("Failed to update campaign".to_string()))?; + modify_campaign(&app.pool, &campaign_being_mutated, &modified_campaign, &app.redis).await.map_err(|_| ResponseError::BadRequest("Failed to update campaign".to_string()))?; Ok(success_response(serde_json::to_string(&campaign)?)) } - pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, modified_campaign: &ModifyCampaign, redis: &MultiplexedConnection) -> Result { + pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, modified_campaign: &ModifyCampaign, redis: &MultiplexedConnection) -> Result { // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 - let new_budget = modified_campaign.budget.ok_or(CampaignError::FailedUpdate("Couldn't get new budget".to_string()))?; - let accounting_spent = get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - - let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - - let old_remaining = get_remaining_for_campaign_from_redis(&redis, campaign.id).await.ok_or(CampaignError::FailedUpdate("Couldn't get remaining for campaign".to_string()))?; + if let Some(new_budget) = modified_campaign.budget { + let old_remaining = get_remaining_for_campaign(&redis, campaign.id).await?.ok_or(Error::FailedUpdate("No remaining entry for campaign".to_string()))?; - let campaign_spent = new_budget.checked_sub(&old_remaining).ok_or(CampaignError::CalculationError)?; - if campaign_spent >= new_budget { - return Err(CampaignError::BudgetExceeded); - } + let campaign_spent = campaign.budget.checked_sub(&old_remaining).ok_or(Error::Calculation)?; + if campaign_spent >= new_budget { + return Err(Error::NewBudget("New budget should be greater than the spent amount".to_string())); + } - let old_remaining = UnifiedNum::from_u64(max(0, old_remaining.to_u64())); + // Separate variable for clarity + let old_budget = campaign.budget; - let new_remaining = new_budget.checked_sub(&campaign_spent).ok_or(CampaignError::CalculationError)?; + match new_budget.cmp(&old_budget) { + Ordering::Greater | Ordering::Equal => { + let new_remaining = old_remaining.checked_add(&new_budget.checked_sub(&old_budget).ok_or(Error::Calculation)?).ok_or(Error::Calculation)?; + let amount_to_incr = new_remaining.checked_sub(&old_remaining).ok_or(Error::Calculation)?; + increase_remaining_for_campaign(&redis, campaign.id, amount_to_incr).await?; + }, + Ordering::Less => { + let new_remaining = old_remaining.checked_add(&old_budget.checked_sub(&new_budget).ok_or(Error::Calculation)?).ok_or(Error::Calculation)?; + let amount_to_decr = new_remaining.checked_sub(&old_remaining).ok_or(Error::Calculation)?; + let decreased_remaining = decrease_remaining_for_campaign(&redis, campaign.id, amount_to_decr).await.ok_or(Error::FailedUpdate("Could't decrease remaining".to_string()))?; + // If it goes below 0 it will still return 0 + if decreased_remaining.eq(&UnifiedNum::from_u64(0)) { + return Err(Error::NewBudget("No budget remaining after decreasing".to_string())); + } + } + } + }; - if new_remaining >= old_remaining { - let diff_in_remaining = new_remaining.checked_sub(&old_remaining).ok_or(CampaignError::CalculationError)?; - increase_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; - } else { - let diff_in_remaining = old_remaining.checked_sub(&new_remaining).ok_or(CampaignError::CalculationError)?; - decrease_remaining_for_campaign(&redis, campaign.id, diff_in_remaining).await?; - } + let accounting_spent = get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; // Gets the latest Spendable for this (spender, channelId) pair - let total_remaining = get_total_remaining_for_channel(&accounting_spent, &latest_spendable).ok_or(CampaignError::FailedUpdate("Could not get total remaining for channel".to_string()))?; + let total_deposited = latest_spendable.deposit.total; + + let total_remaining = total_deposited.checked_sub(&accounting_spent).ok_or(Error::Calculation)?; let campaigns_for_channel = get_campaigns_by_channel(&pool, &campaign.channel.id()).await?; - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, campaign.id, &new_budget).await.map_err(|_| CampaignError::CalculationError)?; + // campaign.budget should 100% exist, therefore it should be safe to just unwrap? + let current_campaign_budget = modified_campaign.budget.ok_or(campaign.budget).unwrap(); + let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, campaign.id, ¤t_campaign_budget).await.map_err(|_| Error::Calculation)?; if campaigns_remaining_sum > total_remaining { - return Err(CampaignError::BudgetExceeded); + return Err(Error::BudgetExceeded); } - update_campaign(&pool, &campaign).await?; + update_campaign(&pool, &campaign, &modified_campaign).await?; Ok(campaign.clone()) } - // TODO: #382 Remove after #412 is merged - fn get_unified_num_from_string(value: &str) -> Option { - let value_as_big_num: Option = BigNum::from_str(value).ok(); - let value_as_u64 = match value_as_big_num { - Some(num) => num.to_u64(), - _ => None, - }; - let value_as_unified = match value_as_u64 { - Some(num) => Some(UnifiedNum::from_u64(num)), - _ => None - }; - value_as_unified - } - - pub async fn get_remaining_for_campaign_from_redis(redis: &MultiplexedConnection, id: CampaignId) -> Option { - let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, id); + pub async fn get_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result, RedisError> { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); let remaining = match redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) - .await - { + .query_async::<_, Option>(&mut redis.clone()) + .await { Ok(Some(remaining)) => { - // TODO: #382 Just parse from string once #412 is merged - get_unified_num_from_string(&remaining) + // Can't be negative due to max() + Some(UnifiedNum::from_u64(max(0, remaining).try_into().unwrap())) }, - _ => None + Ok(None) => None, + Err(e) => return Err(e), }; - remaining + Ok(remaining) } - async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign]) -> Result, CampaignError> { - let keys: Vec = campaigns.into_iter().map(|c| format!("{}:{}", *CAMPAIGN_REMAINING_KEY, c.id)).collect(); + async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign]) -> Result, Error> { + let keys: Vec = campaigns.iter().map(|c| format!("{}:{}", CAMPAIGN_REMAINING_KEY, c.id)).collect(); let remainings = redis::cmd("MGET") .arg(keys) - .query_async::<_, Vec>>(&mut redis.clone()) + .query_async::<_, Vec>>(&mut redis.clone()) .await?; let remainings = remainings .into_iter() - .flat_map(|r| r) - .map(|r| get_unified_num_from_string(&r)) - .flatten() + .map(|r| { + match r { + // Can't be negative due to max() + Some(remaining) => UnifiedNum::from_u64(max(0, remaining).try_into().unwrap()), + None => UnifiedNum::from_u64(0) + } + }) .collect(); Ok(remainings) } - pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: CampaignId, new_budget: &UnifiedNum) -> Result { + pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: CampaignId, new_budget: &UnifiedNum) -> Result { let other_campaigns_remaining = get_remaining_for_multiple_campaigns(&redis, &campaigns).await?; let sum_of_campaigns_remaining = other_campaigns_remaining - .into_iter() - .try_fold(UnifiedNum::from_u64(0), |sum, val| sum.checked_add(&val).ok_or(CampaignError::CalculationError))?; + .iter() + .sum::>() + .ok_or(Error::Calculation)?; // Necessary to do it explicitly for current campaign as its budget is not yet updated in DB - let old_remaining_for_mutated_campaign = get_remaining_for_campaign_from_redis(&redis, mutated_campaign).await.ok_or(CampaignError::CalculationError)?; - let spent_for_mutated_campaign = new_budget.checked_sub(&old_remaining_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; - let new_remaining_for_mutated_campaign = new_budget.checked_sub(&spent_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; - sum_of_campaigns_remaining.checked_add(&new_remaining_for_mutated_campaign).ok_or(CampaignError::CalculationError)?; + let old_remaining_for_mutated_campaign = get_remaining_for_campaign(&redis, mutated_campaign).await?.ok_or(Error::FailedUpdate("No remaining entry for campaign".to_string()))?; + let spent_for_mutated_campaign = new_budget.checked_sub(&old_remaining_for_mutated_campaign).ok_or(Error::Calculation)?; + let new_remaining_for_mutated_campaign = new_budget.checked_sub(&spent_for_mutated_campaign).ok_or(Error::Calculation)?; + sum_of_campaigns_remaining.checked_add(&new_remaining_for_mutated_campaign).ok_or(Error::Calculation)?; Ok(sum_of_campaigns_remaining) } } @@ -336,30 +358,21 @@ async fn process_events( mod test { use primitives::{ util::tests::prep_db::{DUMMY_CAMPAIGN}, - spender::Deposit, + spender::{Deposit, Spendable}, Address }; use deadpool::managed::Object; use crate::{ db::redis_pool::{Manager, TESTS_POOL}, - campaign::update_campaign::CAMPAIGN_REMAINING_KEY, + campaign::update_campaign::{CAMPAIGN_REMAINING_KEY, increase_remaining_for_campaign}, }; use std::convert::TryFrom; use super::*; - async fn get_redis() -> Object { - let connection = TESTS_POOL.get().await.expect("Should return Object"); - connection - } - - fn get_campaign() -> Campaign { - DUMMY_CAMPAIGN.clone() - } - - fn get_dummy_spendable(spender: Address) -> Spendable { + fn get_dummy_spendable(spender: Address, campaign: Campaign) -> Spendable { Spendable { spender, - channel: DUMMY_CAMPAIGN.channel.clone(), + channel: campaign.channel.clone(), deposit: Deposit { total: UnifiedNum::from_u64(1_000_000), still_on_create2: UnifiedNum::from_u64(0), @@ -367,22 +380,11 @@ mod test { } } - #[tokio::test] - async fn does_it_get_total_remaianing() { - let campaign = get_campaign(); - let accounting_spent = UnifiedNum::from_u64(100_000); - let latest_spendable = get_dummy_spendable(campaign.creator); - - let total_remaining = get_total_remaining_for_channel(&accounting_spent, &latest_spendable).expect("should calculate"); - - assert_eq!(total_remaining, UnifiedNum::from_u64(900_000)); - } - #[tokio::test] async fn does_it_increase_remaining() { - let mut redis = get_redis().await; - let campaign = get_campaign(); - let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, campaign.id); + let mut redis = TESTS_POOL.get().await.expect("Should return Object"); + let campaign = DUMMY_CAMPAIGN.clone(); + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign.id); // Setting the redis base variable redis::cmd("SET") @@ -413,15 +415,15 @@ mod test { let remaining = redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.connection) + // Directly parsing to u64 as we know it will be >0 + .query_async::<_, Option>(&mut redis.connection) .await .expect("should get remaining"); assert_eq!(remaining.is_some(), true); - let remaining = remaining.expect("should get remaining"); - let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); - let should_be_remaining = UnifiedNum::from_u64(500).checked_add(&campaign.budget).expect("should add"); - assert_eq!(remaining, should_be_remaining); + let remaining = remaining.expect("should get result out of the option"); + let should_be_remaining = UnifiedNum::from_u64(500) + campaign.budget; + assert_eq!(UnifiedNum::from_u64(remaining), should_be_remaining); increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(0)).await.expect("should work"); @@ -436,25 +438,4 @@ mod test { let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); assert_eq!(remaining, should_be_remaining); } - - #[tokio::test] - async fn update_remaining_before_it_is_set() { - let mut redis = get_redis().await; - let campaign = get_campaign(); - let key = format!("{}:{}", *CAMPAIGN_REMAINING_KEY, campaign.id); - - let remaining = redis::cmd("GET") - .arg(&key) - .query_async::<_, Option>(&mut redis.connection) - .await - .expect("should return None"); - - assert_eq!(remaining, None) - } - - // test get_campaigns_remaining_sum - - // test get_remaining_for_multiple_campaigns - - // test get_remaining_for_campaign_from_redis } \ No newline at end of file From 5cb26efc6b5b71c612312d668f8b817e765ae376 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Fri, 9 Jul 2021 09:04:38 +0300 Subject: [PATCH 29/49] sentry - prefix campaign routes with `/v5/` --- sentry/src/lib.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index cb789481c..9289adad8 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -64,10 +64,10 @@ lazy_static! { } static INSERT_EVENTS_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { - Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/events/?$").expect("The regex should be valid") + Regex::new(r"^/v5/campaign/0x([a-zA-Z0-9]{32})/events/?$").expect("The regex should be valid") }); static CLOSE_CAMPAIGN_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { - Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/close/?$").expect("The regex should be valid") + Regex::new(r"^/v5/campaign/0x([a-zA-Z0-9]{32})/close/?$").expect("The regex should be valid") }); #[derive(Debug, Clone)] @@ -162,7 +162,7 @@ impl Application { // This is important because it prevents us from doing // expensive regex matching for routes without /channel (path, _) if path.starts_with("/channel") => channels_router(req, &self).await, - (path, _) if path.starts_with("/campaign") => campaigns_router(req, &self).await, + (path, _) if path.starts_with("/v5/campaign") => campaigns_router(req, &self).await, _ => Err(ResponseError::NotFound), } .unwrap_or_else(map_response_error); @@ -217,8 +217,6 @@ async fn analytics_router( ) -> Result, ResponseError> { let (route, method) = (req.uri().path(), req.method()); - - // TODO AIP#61: Add routes for: // - POST /channel/:id/pay // #[serde(rename_all = "camelCase")] @@ -295,9 +293,8 @@ async fn channels_router( req.extensions_mut().insert(param); insert_events(req, app).await - } else */ - if let (Some(caps), &Method::GET) = (LAST_APPROVED_BY_CHANNEL_ID.captures(&path), method) - { + } else */ + if let (Some(caps), &Method::GET) = (LAST_APPROVED_BY_CHANNEL_ID.captures(&path), method) { let param = RouteParams(vec![caps .get(1) .map_or("".to_string(), |m| m.as_str().to_string())]); From 4f238614917f18355cd06c6c54f26eece0203d44 Mon Sep 17 00:00:00 2001 From: simzzz Date: Fri, 9 Jul 2021 21:23:58 +0300 Subject: [PATCH 30/49] PR changes --- primitives/src/sentry.rs | 18 ++++++++++ sentry/src/db/campaign.rs | 47 ++++++++---------------- sentry/src/lib.rs | 29 ++++----------- sentry/src/routes/campaign.rs | 68 +++++++++++++++++------------------ 4 files changed, 72 insertions(+), 90 deletions(-) diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index cb2fe99c2..862a1eaef 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -389,6 +389,24 @@ pub mod campaign_create { targeting_rules: Some(campaign.targeting_rules), } } + + pub fn apply(&self, campaign: &Campaign) -> Campaign { + let campaign = campaign.clone(); + Campaign { + id: campaign.id, + channel: campaign.channel, + creator: campaign.creator, + budget: self.budget.unwrap_or(campaign.budget), + validators: self.validators.clone().unwrap_or(campaign.validators), + title: self.title.clone().or(campaign.title), + pricing_bounds: self.pricing_bounds.clone().or(campaign.pricing_bounds), + event_submission: self.event_submission.clone().or(campaign.event_submission), + ad_units: self.ad_units.clone().unwrap_or(campaign.ad_units), + targeting_rules: self.targeting_rules.clone().unwrap_or(campaign.targeting_rules), + created: campaign.created, + active: campaign.active, + } + } } } diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 28b04a41e..75989a420 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -1,5 +1,5 @@ use crate::db::{DbPool, PoolError}; -use primitives::{sentry::campaign_create::ModifyCampaign, Campaign, CampaignId, ChannelId}; +use primitives::{Campaign, CampaignId, ChannelId}; use tokio_postgres::types::Json; pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result { @@ -48,6 +48,7 @@ pub async fn fetch_campaign( Ok(row.as_ref().map(Campaign::from)) } +// TODO: We might need to use LIMIT to implement pagination pub async fn get_campaigns_by_channel( pool: &DbPool, channel_id: &ChannelId, @@ -65,48 +66,26 @@ pub async fn get_campaigns_by_channel( pub async fn update_campaign( pool: &DbPool, campaign: &Campaign, - modified_campaign: &ModifyCampaign, ) -> Result { let client = pool.get().await?; let statement = client .prepare("UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 WHERE id = $8") .await?; - let ad_units = &modified_campaign - .ad_units - .as_ref() - .unwrap_or(&campaign.ad_units); - let ad_units = Json(ad_units); + + let ad_units = Json(&campaign.ad_units); let updated_rows = client .execute( &statement, &[ - &modified_campaign.budget.unwrap_or(campaign.budget), - &modified_campaign - .validators - .as_ref() - .unwrap_or(&campaign.validators), - &modified_campaign - .title - .as_ref() - .ok_or_else(|| &campaign.title) - .unwrap(), - &modified_campaign - .pricing_bounds - .as_ref() - .ok_or_else(|| &campaign.title) - .unwrap(), - &modified_campaign - .event_submission - .as_ref() - .ok_or_else(|| &campaign.title) - .unwrap(), + &campaign.budget, + &campaign.validators, + &campaign.title, + &campaign.pricing_bounds, + &campaign.event_submission, &ad_units, - &modified_campaign - .targeting_rules - .as_ref() - .unwrap_or(&campaign.targeting_rules), + &campaign.targeting_rules, &campaign.id, ], ) @@ -121,6 +100,7 @@ mod test { use primitives::{ util::tests::prep_db::{DUMMY_CAMPAIGN, DUMMY_AD_UNITS}, event_submission::{Rule, RateLimit}, + sentry::campaign_create::ModifyCampaign, targeting::Rules, UnifiedNum, EventSubmission, }; @@ -130,7 +110,6 @@ mod test { use crate::{ db::tests_postgres::{setup_test_migrations, DATABASE_POOL}, - ResponseError, }; use super::*; @@ -209,7 +188,9 @@ mod test { targeting_rules: Some(Rules::new()), }; - let is_campaign_updated = update_campaign(&database.pool, &campaign_for_testing, &modified_campaign).await.expect("should update"); + let applied_campaign = modified_campaign.apply(&campaign_for_testing); + + let is_campaign_updated = update_campaign(&database.pool, &applied_campaign).await.expect("should update"); assert!(is_campaign_updated); } } diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index f98886b5a..e16be4ac3 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::all)] #![deny(rust_2018_idioms)] -use crate::db::{fetch_campaign, DbPool}; +use crate::db::{DbPool}; use crate::routes::campaign; use crate::routes::channel::channel_status; use crate::routes::event_aggregate::list_channel_event_aggregates; @@ -18,7 +18,7 @@ use middleware::{campaign::CampaignLoad, Chain, Middleware}; use once_cell::sync::Lazy; use primitives::adapter::Adapter; use primitives::sentry::ValidationErrorResponse; -use primitives::{CampaignId, Config, ValidatorId}; +use primitives::{Config, ValidatorId}; use redis::aio::MultiplexedConnection; use regex::Regex; use routes::analytics::{advanced_analytics, advertiser_analytics, analytics, publisher_analytics}; @@ -29,7 +29,6 @@ use routes::channel::{ }; use slog::Logger; use std::collections::HashMap; -use std::str::FromStr; pub mod middleware; pub mod routes { @@ -69,10 +68,10 @@ static INSERT_EVENTS_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/events/?$").expect("The regex should be valid") }); static CLOSE_CAMPAIGN_BY_CAMPAIGN_ID: Lazy = Lazy::new(|| { - Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/close/?$").expect("The regex should be valid") + Regex::new(r"^/v5/campaign/0x([a-zA-Z0-9]{32})/close/?$").expect("The regex should be valid") }); static CAMPAIGN_UPDATE_BY_ID: Lazy = Lazy::new(|| { - Regex::new(r"^/campaign/0x([a-zA-Z0-9]{32})/?$").expect("The regex should be valid") + Regex::new(r"^/v5/campaign/0x([a-zA-Z0-9]{32})/?$").expect("The regex should be valid") }); #[derive(Debug, Clone)] @@ -164,7 +163,7 @@ impl Application { publisher_analytics(req, &self).await } // For creating campaigns - ("/campaign", &Method::POST) => { + ("/v5/campaign", &Method::POST) => { let req = match AuthRequired.call(req, &self).await { Ok(req) => req, Err(error) => { @@ -195,23 +194,9 @@ async fn campaigns_router( ) -> Result, ResponseError> { let (path, method) = (req.uri().path(), req.method()); - if let (Some(caps), &Method::POST) = (CAMPAIGN_UPDATE_BY_ID.captures(&path), method) { - let param = RouteParams(vec![caps - .get(1) - .map_or("".to_string(), |m| m.as_str().to_string())]); - - // Should be safe to access indice here - let campaign_id = param - .get(0) - .ok_or_else(|| ResponseError::BadRequest("No CampaignId".to_string()))?; - let campaign_id = CampaignId::from_str(&campaign_id) - .map_err(|_| ResponseError::BadRequest("Bad CampaignId".to_string()))?; - - let campaign = fetch_campaign(app.pool.clone(), &campaign_id) - .await - .map_err(|_| ResponseError::NotFound)?; + if let (Some(_caps), &Method::POST) = (CAMPAIGN_UPDATE_BY_ID.captures(&path), method) { + let req = CampaignLoad.call(req, app).await?; - req.extensions_mut().insert(campaign); update_campaign::handle_route(req, app).await } else if let (Some(caps), &Method::POST) = (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 8b0e5867a..e37502c91 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -19,7 +19,7 @@ use primitives::{ Event, SuccessResponse, }, campaign_validator::Validator, - Address, Campaign, CampaignId, UnifiedNum + Campaign, CampaignId, UnifiedNum }; use redis::{ aio::MultiplexedConnection, @@ -57,7 +57,7 @@ pub async fn create_campaign( req: Request, app: &Application, ) -> Result, ResponseError> { - let session = req + let auth = req .extensions() .get::() .expect("request should have session") @@ -72,8 +72,7 @@ pub async fn create_campaign( campaign.validate(&app.config, &app.adapter.whoami()).map_err(|_| ResponseError::FailedValidation("couldn't valdiate campaign".to_string()))?; - // TODO: Just use session.uid once it's address - if Address::from_bytes(session.uid.as_bytes()) != campaign.creator { + if auth.uid.to_address() != campaign.creator { return Err(ResponseError::Forbidden("Request not sent by campaign creator".to_string())) } @@ -85,14 +84,14 @@ pub async fn create_campaign( let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; let total_deposited = latest_spendable.deposit.total; - let remaining_for_channel = total_deposited.checked_sub(&accounting_spent).ok_or(ResponseError::FailedValidation("couldn't calculate remaining for channel".to_string()))?; + let remaining_for_channel = total_deposited.checked_sub(&accounting_spent).ok_or(ResponseError::FailedValidation("No more budget remaining".to_string()))?; if campaign.budget > remaining_for_channel { - return Err(ResponseError::BadRequest("Not Enough budget for campaign".to_string())); + return Err(ResponseError::BadRequest("Not enough deposit left for the new campaign budget".to_string())); } // If the campaign is being created, the amount spent is 0, therefore remaining = budget - set_initial_remaining_for_campaign(&app.redis.clone(), campaign.id, campaign.budget).await.map_err(|_| ResponseError::BadRequest("Couldn't update remaining while creating campaign".to_string()))?; + set_initial_remaining_for_campaign(&mut app.redis.clone(), campaign.id, campaign.budget).await.map_err(|_| ResponseError::BadRequest("Couldn't update remaining while creating campaign".to_string()))?; // insert Campaign match insert_campaign(&app.pool, &campaign).await { @@ -116,16 +115,15 @@ pub async fn create_campaign( pub mod update_campaign { use super::*; - use crate::fetch_campaign; pub const CAMPAIGN_REMAINING_KEY: &'static str = "campaignRemaining"; - pub async fn set_initial_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + pub async fn set_initial_remaining_for_campaign(redis: &mut MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); redis::cmd("SETNX") .arg(&key) .arg(amount.to_u64()) - .query_async(&mut redis.clone()) + .query_async(redis) .await?; Ok(true) } @@ -161,16 +159,19 @@ pub mod update_campaign { req: Request, app: &Application, ) -> Result, ResponseError> { - let campaign = req.extensions().get::().expect("We must have a campaign in extensions"); - let modified_campaign = ModifyCampaign::from_campaign(campaign.clone()); + let campaign_being_mutated = req.extensions().get::().expect("We must have a campaign in extensions").to_owned(); + + let body = hyper::body::to_bytes(req.into_body()).await?; + + let modified_campaign = serde_json::from_slice::(&body) + .map_err(|e| ResponseError::FailedValidation(e.to_string()))?; - // Getting the campaign as it was before the update operation, should exist - let campaign_being_mutated = fetch_campaign(app.pool.clone(), &campaign.id).await?.ok_or(ResponseError::NotFound)?; + let modified_campaign = ModifyCampaign::from_campaign(modified_campaign.clone()); // modify Campaign modify_campaign(&app.pool, &campaign_being_mutated, &modified_campaign, &app.redis).await.map_err(|_| ResponseError::BadRequest("Failed to update campaign".to_string()))?; - Ok(success_response(serde_json::to_string(&campaign)?)) + Ok(success_response(serde_json::to_string(&modified_campaign)?)) } pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, modified_campaign: &ModifyCampaign, redis: &MultiplexedConnection) -> Result { @@ -190,7 +191,8 @@ pub mod update_campaign { let old_budget = campaign.budget; match new_budget.cmp(&old_budget) { - Ordering::Greater | Ordering::Equal => { + Ordering::Equal => (), + Ordering::Greater => { let new_remaining = old_remaining.checked_add(&new_budget.checked_sub(&old_budget).ok_or(Error::Calculation)?).ok_or(Error::Calculation)?; let amount_to_incr = new_remaining.checked_sub(&old_remaining).ok_or(Error::Calculation)?; increase_remaining_for_campaign(&redis, campaign.id, amount_to_incr).await?; @@ -216,15 +218,13 @@ pub mod update_campaign { let total_remaining = total_deposited.checked_sub(&accounting_spent).ok_or(Error::Calculation)?; let campaigns_for_channel = get_campaigns_by_channel(&pool, &campaign.channel.id()).await?; - // campaign.budget should 100% exist, therefore it should be safe to just unwrap? - let current_campaign_budget = modified_campaign.budget.ok_or(campaign.budget).unwrap(); + let current_campaign_budget = modified_campaign.budget.unwrap_or(campaign.budget); let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, campaign.id, ¤t_campaign_budget).await.map_err(|_| Error::Calculation)?; - if campaigns_remaining_sum > total_remaining { - return Err(Error::BudgetExceeded); + if campaigns_remaining_sum <= total_remaining { + let campaign_with_updates = modified_campaign.apply(campaign); + update_campaign(&pool, &campaign_with_updates).await?; } - update_campaign(&pool, &campaign, &modified_campaign).await?; - Ok(campaign.clone()) } @@ -361,24 +361,22 @@ mod test { spender::{Deposit, Spendable}, Address }; - use deadpool::managed::Object; use crate::{ - db::redis_pool::{Manager, TESTS_POOL}, + db::redis_pool::TESTS_POOL, campaign::update_campaign::{CAMPAIGN_REMAINING_KEY, increase_remaining_for_campaign}, }; - use std::convert::TryFrom; use super::*; - fn get_dummy_spendable(spender: Address, campaign: Campaign) -> Spendable { - Spendable { - spender, - channel: campaign.channel.clone(), - deposit: Deposit { - total: UnifiedNum::from_u64(1_000_000), - still_on_create2: UnifiedNum::from_u64(0), - }, - } - } + // fn get_dummy_spendable(spender: Address, campaign: Campaign) -> Spendable { + // Spendable { + // spender, + // channel: campaign.channel.clone(), + // deposit: Deposit { + // total: UnifiedNum::from_u64(1_000_000), + // still_on_create2: UnifiedNum::from_u64(0), + // }, + // } + // } #[tokio::test] async fn does_it_increase_remaining() { From be363277ad05f85c03d60badbc1845efb06ee5af Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 12 Jul 2021 09:54:39 +0300 Subject: [PATCH 31/49] sentry - db - accounting - spend_amount with delta Balances --- primitives/src/campaign.rs | 2 +- sentry/src/db/accounting.rs | 364 +++++++++++++++++++++++++++++------- 2 files changed, 301 insertions(+), 65 deletions(-) diff --git a/primitives/src/campaign.rs b/primitives/src/campaign.rs index bbbb5b3c4..ad3ca0167 100644 --- a/primitives/src/campaign.rs +++ b/primitives/src/campaign.rs @@ -195,7 +195,7 @@ impl Campaign { } } - /// Matches the Channel.leader to the Campaign.spec.leader + /// Matches the Channel.leader to the Campaign.validators.leader /// If they match it returns `Some`, otherwise, it returns `None` pub fn leader(&self) -> Option<&'_ ValidatorDesc> { self.validators.find(&self.channel.leader) diff --git a/sentry/src/db/accounting.rs b/sentry/src/db/accounting.rs index fe1f0b0d4..db1588994 100644 --- a/sentry/src/db/accounting.rs +++ b/sentry/src/db/accounting.rs @@ -1,8 +1,12 @@ use chrono::{DateTime, Utc}; use primitives::{ + sentry::accounting::{Balances, CheckedState}, Address, ChannelId, UnifiedNum, }; -use tokio_postgres::{IsolationLevel, Row, types::{FromSql, ToSql}}; +use tokio_postgres::{ + types::{FromSql, ToSql}, + IsolationLevel, Row, +}; use super::{DbPool, PoolError}; use thiserror::Error; @@ -15,6 +19,12 @@ pub enum Error { Postgres(#[from] PoolError), } +impl From for Error { + fn from(error: tokio_postgres::Error) -> Self { + Self::Postgres(PoolError::Backend(error)) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Accounting { pub channel_id: ChannelId, @@ -83,9 +93,7 @@ pub async fn update_accounting( amount: UnifiedNum, ) -> Result { let client = pool.get().await?; - let statement = client - .prepare(UPDATE_ACCOUNTING_STATEMENT) - .await?; + let statement = client.prepare(UPDATE_ACCOUNTING_STATEMENT).await?; let now = Utc::now(); let updated: Option> = None; @@ -100,32 +108,64 @@ pub async fn update_accounting( Ok(Accounting::from(&row)) } -/// Will use `UPDATE_ACCOUNTING_STATEMENT` to create and run the query twice - once for Earner and once for Spender accounting. -/// -/// It runs both queries in a transaction in order to rollback if one of the queries fails. -pub async fn spend_accountings( +/// `delta_balances` defines the Balances that need to be added to the spending or earnings of the `Accounting`s. +/// It will **not** override the whole `Accounting` value +/// Returns a tuple of `(Vec, Vec)` +pub async fn spend_amount( pool: DbPool, channel_id: ChannelId, - earner: Address, - spender: Address, - amount: UnifiedNum, -) -> Result<(Accounting, Accounting), PoolError> { + delta_balances: Balances, +) -> Result<(Vec, Vec), PoolError> { let mut client = pool.get().await?; // The reads and writes in this transaction must be able to be committed as an atomic “unit” with respect to reads and writes of all other concurrent serializable transactions without interleaving. - let transaction = client.build_transaction().isolation_level(IsolationLevel::Serializable).start().await?; + let transaction = client + .build_transaction() + .isolation_level(IsolationLevel::Serializable) + .start() + .await?; let statement = transaction.prepare(UPDATE_ACCOUNTING_STATEMENT).await?; let now = Utc::now(); let updated: Option> = None; - let earner_row = transaction.query_one(&statement, &[&channel_id, &Side::Earner, &earner, &amount, &updated, &now]).await?; - let spender_row = transaction.query_one(&statement, &[&channel_id, &Side::Spender, &spender, &amount, &updated, &now]).await?; + let (mut earners, mut spenders) = (vec![], vec![]); + + // Earners + for (earner, amount) in delta_balances.earners { + let row = transaction + .query_one( + &statement, + &[&channel_id, &Side::Earner, &earner, &amount, &updated, &now], + ) + .await?; + + earners.push(Accounting::from(&row)) + } + + // Spenders + for (spender, amount) in delta_balances.spenders { + let row = transaction + .query_one( + &statement, + &[ + &channel_id, + &Side::Spender, + &spender, + &amount, + &updated, + &now, + ], + ) + .await?; + + spenders.push(Accounting::from(&row)) + } transaction.commit().await?; - Ok((Accounting::from(&earner_row), Accounting::from(&spender_row))) + Ok((earners, spenders)) } #[cfg(test)] @@ -183,10 +223,14 @@ mod test { "Should add the newly spent amount to the existing one" ); - let spent = get_accounting(database.pool.clone(), channel_id, spender, Side::Spender).await.expect("Should query for the updated accounting"); + let spent = get_accounting(database.pool.clone(), channel_id, spender, Side::Spender) + .await + .expect("Should query for the updated accounting"); assert_eq!(Some(updated), spent); - let earned = get_accounting(database.pool.clone(), channel_id, spender, Side::Earner).await.expect("Should query for accounting"); + let earned = get_accounting(database.pool.clone(), channel_id, spender, Side::Earner) + .await + .expect("Should query for accounting"); assert!(earned.is_none(), "Spender shouldn't have an earned amount"); } @@ -222,14 +266,17 @@ mod test { "Should add the newly earned amount to the existing one" ); - let earned = get_accounting(database.pool.clone(), channel_id, earner, Side::Earner).await.expect("Should query for the updated accounting"); + let earned = get_accounting(database.pool.clone(), channel_id, earner, Side::Earner) + .await + .expect("Should query for the updated accounting"); assert_eq!(Some(updated), earned); - let spent = get_accounting(database.pool.clone(), channel_id, earner, Side::Spender).await.expect("Should query for accounting"); + let spent = get_accounting(database.pool.clone(), channel_id, earner, Side::Spender) + .await + .expect("Should query for accounting"); assert!(spent.is_none(), "Earner shouldn't have a spent amount"); } - // Spender as Earner & another Spender // Will test the previously spent amount as well! { @@ -265,17 +312,64 @@ mod test { "Should add the newly spent amount to the existing one" ); - let earned_acc = get_accounting(database.pool.clone(), channel_id, spender_as_earner, Side::Earner).await.expect("Should query for earned accounting").expect("Should have Earned accounting for Spender as Earner"); + let earned_acc = get_accounting( + database.pool.clone(), + channel_id, + spender_as_earner, + Side::Earner, + ) + .await + .expect("Should query for earned accounting") + .expect("Should have Earned accounting for Spender as Earner"); assert_eq!(UnifiedNum::from(100_000_999), earned_acc.amount); - - let spent_acc = get_accounting(database.pool.clone(), channel_id, spender_as_earner, Side::Spender).await.expect("Should query for spent accounting").expect("Should have Spent accounting for Spender as Earner"); + + let spent_acc = get_accounting( + database.pool.clone(), + channel_id, + spender_as_earner, + Side::Spender, + ) + .await + .expect("Should query for spent accounting") + .expect("Should have Spent accounting for Spender as Earner"); assert_eq!(UnifiedNum::from(300_000_000), spent_acc.amount); - } } - + + fn assert_accounting( + expected: (Address, Side, UnifiedNum), + accounting: Accounting, + with_set_updated: bool, + // ) -> anyhow::Result<()> { + ) { + assert_eq!( + expected.0, accounting.address, + "Accounting address is not the same" + ); + assert_eq!( + expected.1, accounting.side, + "Accounting side is not the same" + ); + assert_eq!( + expected.2, accounting.amount, + "Accounting amount is not the same" + ); + + if with_set_updated { + assert!( + accounting.updated.is_some(), + "Accounting should have been updated" + ) + } else { + assert!( + accounting.updated.is_none(), + "Accounting should not have been updated" + ) + } + } + #[tokio::test] - async fn test_spending_accountings() { + async fn test_spend_amount() { let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); setup_test_migrations(database.pool.clone()) @@ -285,47 +379,189 @@ mod test { let channel_id = DUMMY_CAMPAIGN.channel.id(); let earner = ADDRESSES["publisher"]; let spender = ADDRESSES["creator"]; + let spender_as_earner = spender; let other_spender = ADDRESSES["tester"]; - let amount = UnifiedNum::from(100_000_000); - let update_amount = UnifiedNum::from(200_000_000); + let cases = [ + // Spender & Earner insert + ( + UnifiedNum::from(100_000_000), + earner, + spender, + [ + vec![(earner, Side::Earner, UnifiedNum::from(100_000_000), false)], + vec![(spender, Side::Spender, UnifiedNum::from(100_000_000), false)], + ], + ), + // Spender & Earner update + ( + UnifiedNum::from(200_000_000), + earner, + spender, + [ + vec![(earner, Side::Earner, UnifiedNum::from(300_000_000), true)], + vec![(spender, Side::Spender, UnifiedNum::from(300_000_000), true)], + ], + ), + // Spender as an Earner & another spender + ( + UnifiedNum::from(999), + spender_as_earner, + other_spender, + [ + vec![(spender, Side::Earner, UnifiedNum::from(999), false)], + vec![(other_spender, Side::Spender, UnifiedNum::from(999), false)], + ], + ), + ]; + + for (amount_to_spend, earner, spender, [earners, spenders]) in cases { + // Spender & Earner insert + let mut balances = Balances::::default(); + balances + .spend(spender, earner, amount_to_spend) + .expect("Should spend"); + + let (actual_earners, actual_spenders) = + spend_amount(database.pool.clone(), channel_id, balances) + .await + .expect("Should insert Earner and Spender"); + + for (actual, expected) in actual_earners.into_iter().zip(earners) { + assert_accounting((expected.0, expected.1, expected.2), actual, expected.3) + } + + for (actual, expected) in actual_spenders.into_iter().zip(spenders) { + assert_accounting((expected.0, expected.1, expected.2), actual, expected.3) + } + } - // Spender & Earner insert - let (inserted_earner, inserted_spender) = spend_accountings(database.pool.clone(), channel_id, earner, spender, amount).await.expect("Should insert Earner and Spender"); - assert_eq!(earner, inserted_earner.address); - assert_eq!(Side::Earner, inserted_earner.side); - assert_eq!(UnifiedNum::from(100_000_000), inserted_earner.amount); - - assert_eq!(spender, inserted_spender.address); - assert_eq!(Side::Spender, inserted_spender.side); - assert_eq!(UnifiedNum::from(100_000_000), inserted_spender.amount); - - // Spender & Earner update - let (updated_earner, updated_spender) = spend_accountings(database.pool.clone(), channel_id, earner, spender, update_amount).await.expect("Should update Earner and Spender"); - - assert_eq!(earner, updated_earner.address); - assert_eq!(Side::Earner, updated_earner.side); - assert_eq!(UnifiedNum::from(300_000_000), updated_earner.amount, "Should add the newly earned amount to the existing one"); - - assert_eq!(spender, updated_spender.address); - assert_eq!(Side::Spender, updated_spender.side); - assert_eq!(UnifiedNum::from(300_000_000), updated_spender.amount, "Should add the newly spend amount to the existing one"); - - // Spender as an Earner & another spender - let (spender_as_earner, inserted_other_spender) = spend_accountings(database.pool.clone(), channel_id, spender, other_spender, UnifiedNum::from(999)).await.expect("Should update Spender as Earner and the Other Spender"); - - assert_eq!(spender, spender_as_earner.address); - assert_eq!(Side::Earner, spender_as_earner.side); - assert_eq!(UnifiedNum::from(999), spender_as_earner.amount, "Should add earner accounting for the previous Spender"); - - assert_eq!(other_spender, inserted_other_spender.address); - assert_eq!(Side::Spender, inserted_other_spender.side); - assert_eq!(UnifiedNum::from(999), inserted_other_spender.amount); - - let earned = get_accounting(database.pool.clone(), channel_id, spender, Side::Earner).await.expect("Should query for accounting").expect("Should have Earned accounting for Spender as Earner"); + // Check the final amounts of Spent/Earned for the Spender + let earned = get_accounting(database.pool.clone(), channel_id, spender, Side::Earner) + .await + .expect("Should query for accounting") + .expect("Should have Earned accounting for Spender as Earner"); assert_eq!(UnifiedNum::from(999), earned.amount); - - let spent = get_accounting(database.pool.clone(), channel_id, spender, Side::Spender).await.expect("Should query for accounting").expect("Should have Spent accounting for Spender as Earner"); + + let spent = get_accounting(database.pool.clone(), channel_id, spender, Side::Spender) + .await + .expect("Should query for accounting") + .expect("Should have Spent accounting for Spender as Earner"); assert_eq!(UnifiedNum::from(300_000_000), spent.amount); } + + #[tokio::test] + async fn test_spend_amount_with_multiple_spends() { + let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); + + setup_test_migrations(database.pool.clone()) + .await + .expect("Migrations should succeed"); + + let channel_id = DUMMY_CAMPAIGN.channel.id(); + let earner = ADDRESSES["publisher"]; + let other_earner = ADDRESSES["publisher2"]; + let spender = ADDRESSES["creator"]; + let spender_as_earner = spender; + let other_spender = ADDRESSES["tester"]; + let third_spender = ADDRESSES["user"]; + + // Spenders & Earners insert + { + let mut balances = Balances::::default(); + balances + .spend(spender, earner, UnifiedNum::from(400_000)) + .expect("Should spend"); + balances + .spend(other_spender, other_earner, UnifiedNum::from(500_000)) + .expect("Should spend"); + + let (earners_acc, spenders_acc) = + spend_amount(database.pool.clone(), channel_id, balances) + .await + .expect("Should insert Earners and Spenders"); + + assert_eq!(2, earners_acc.len()); + assert_eq!(2, spenders_acc.len()); + + // Earners assertions + assert_accounting( + (earner, Side::Earner, UnifiedNum::from(400_000)), + earners_acc.iter().find(|a| a.address == earner).unwrap().clone(), + false, + ); + assert_accounting( + (other_earner, Side::Earner, UnifiedNum::from(500_000)), + earners_acc.iter().find(|a| a.address == other_earner).unwrap().clone(), + false, + ); + + // Spenders assertions + assert_accounting( + (spender, Side::Spender, UnifiedNum::from(400_000)), + spenders_acc.iter().find(|a| a.address == spender).unwrap().clone(), + false, + ); + assert_accounting( + (other_spender, Side::Spender, UnifiedNum::from(500_000)), + spenders_acc.iter().find(|a| a.address == other_spender).unwrap().clone(), + false, + ); + } + // Spenders & Earners update with 1 insert (third_spender & spender_as_earner) + { + let mut balances = Balances::::default(); + balances + .spend(spender, earner, UnifiedNum::from(1_400_000)) + .expect("Should spend"); + balances + .spend(other_spender, other_earner, UnifiedNum::from(1_500_000)) + .expect("Should spend"); + balances + .spend(third_spender, spender_as_earner, UnifiedNum::from(600_000)) + .expect("Should spend"); + + let (earners_acc, spenders_acc) = + spend_amount(database.pool.clone(), channel_id, balances) + .await + .expect("Should update & insert new Earners and Spenders"); + + assert_eq!(3, earners_acc.len()); + assert_eq!(3, spenders_acc.len()); + + // Earners assertions + assert_accounting( + (earner, Side::Earner, UnifiedNum::from(1_800_000)), + earners_acc.iter().find(|a| a.address == earner).unwrap().clone(), + true, + ); + assert_accounting( + (other_earner, Side::Earner, UnifiedNum::from(2_000_000)), + earners_acc.iter().find(|a| a.address == other_earner).unwrap().clone(), + true, + ); + assert_accounting( + (spender_as_earner, Side::Earner, UnifiedNum::from(600_000)), + earners_acc.iter().find(|a| a.address == spender_as_earner).unwrap().clone(), + false, + ); + + // Spenders assertions + assert_accounting( + (spender, Side::Spender, UnifiedNum::from(1_800_000)), + spenders_acc.iter().find(|a| a.address == spender).unwrap().clone(), + true, + ); + assert_accounting( + (other_spender, Side::Spender, UnifiedNum::from(2_000_000)), + spenders_acc.iter().find(|a| a.address == other_spender).unwrap().clone(), + true, + ); + assert_accounting( + (third_spender, Side::Spender, UnifiedNum::from(600_000)), + spenders_acc.iter().find(|a| a.address == third_spender).unwrap().clone(), + false, + ); + } + } } From 6f51c7098f0978c546c0c2d45dbe51ed5da06c77 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 12 Jul 2021 10:08:39 +0300 Subject: [PATCH 32/49] sentry - routes - insert_events route - run rustfmt & fix clippy warnings --- primitives/src/sentry/accounting.rs | 4 +- sentry/src/db/accounting.rs | 61 +++- sentry/src/lib.rs | 4 +- sentry/src/payout.rs | 4 +- sentry/src/routes/campaign.rs | 473 +++++++++++++++++++++++----- sentry/src/spender.rs | 30 +- 6 files changed, 467 insertions(+), 109 deletions(-) diff --git a/primitives/src/sentry/accounting.rs b/primitives/src/sentry/accounting.rs index 64589ce23..867397659 100644 --- a/primitives/src/sentry/accounting.rs +++ b/primitives/src/sentry/accounting.rs @@ -59,12 +59,12 @@ impl Balances { let spent = self.spenders.entry(spender).or_default(); *spent = spent .checked_add(&amount) - .ok_or_else(|| OverflowError::Spender(spender))?; + .ok_or(OverflowError::Spender(spender))?; let earned = self.earners.entry(earner).or_default(); *earned = earned .checked_add(&amount) - .ok_or_else(|| OverflowError::Earner(earner))?; + .ok_or(OverflowError::Earner(earner))?; Ok(()) } diff --git a/sentry/src/db/accounting.rs b/sentry/src/db/accounting.rs index db1588994..ae79ac7bd 100644 --- a/sentry/src/db/accounting.rs +++ b/sentry/src/db/accounting.rs @@ -340,7 +340,6 @@ mod test { expected: (Address, Side, UnifiedNum), accounting: Accounting, with_set_updated: bool, - // ) -> anyhow::Result<()> { ) { assert_eq!( expected.0, accounting.address, @@ -487,24 +486,40 @@ mod test { // Earners assertions assert_accounting( (earner, Side::Earner, UnifiedNum::from(400_000)), - earners_acc.iter().find(|a| a.address == earner).unwrap().clone(), + earners_acc + .iter() + .find(|a| a.address == earner) + .unwrap() + .clone(), false, ); assert_accounting( (other_earner, Side::Earner, UnifiedNum::from(500_000)), - earners_acc.iter().find(|a| a.address == other_earner).unwrap().clone(), + earners_acc + .iter() + .find(|a| a.address == other_earner) + .unwrap() + .clone(), false, ); // Spenders assertions assert_accounting( (spender, Side::Spender, UnifiedNum::from(400_000)), - spenders_acc.iter().find(|a| a.address == spender).unwrap().clone(), + spenders_acc + .iter() + .find(|a| a.address == spender) + .unwrap() + .clone(), false, ); assert_accounting( (other_spender, Side::Spender, UnifiedNum::from(500_000)), - spenders_acc.iter().find(|a| a.address == other_spender).unwrap().clone(), + spenders_acc + .iter() + .find(|a| a.address == other_spender) + .unwrap() + .clone(), false, ); } @@ -532,34 +547,58 @@ mod test { // Earners assertions assert_accounting( (earner, Side::Earner, UnifiedNum::from(1_800_000)), - earners_acc.iter().find(|a| a.address == earner).unwrap().clone(), + earners_acc + .iter() + .find(|a| a.address == earner) + .unwrap() + .clone(), true, ); assert_accounting( (other_earner, Side::Earner, UnifiedNum::from(2_000_000)), - earners_acc.iter().find(|a| a.address == other_earner).unwrap().clone(), + earners_acc + .iter() + .find(|a| a.address == other_earner) + .unwrap() + .clone(), true, ); assert_accounting( (spender_as_earner, Side::Earner, UnifiedNum::from(600_000)), - earners_acc.iter().find(|a| a.address == spender_as_earner).unwrap().clone(), + earners_acc + .iter() + .find(|a| a.address == spender_as_earner) + .unwrap() + .clone(), false, ); // Spenders assertions assert_accounting( (spender, Side::Spender, UnifiedNum::from(1_800_000)), - spenders_acc.iter().find(|a| a.address == spender).unwrap().clone(), + spenders_acc + .iter() + .find(|a| a.address == spender) + .unwrap() + .clone(), true, ); assert_accounting( (other_spender, Side::Spender, UnifiedNum::from(2_000_000)), - spenders_acc.iter().find(|a| a.address == other_spender).unwrap().clone(), + spenders_acc + .iter() + .find(|a| a.address == other_spender) + .unwrap() + .clone(), true, ); assert_accounting( (third_spender, Side::Spender, UnifiedNum::from(600_000)), - spenders_acc.iter().find(|a| a.address == third_spender).unwrap().clone(), + spenders_acc + .iter() + .find(|a| a.address == third_spender) + .unwrap() + .clone(), false, ); } diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 9289adad8..71c56bba4 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -102,10 +102,10 @@ impl Application { ) -> Self { Self { adapter, - config, logger, redis, pool, + config, } } @@ -188,7 +188,7 @@ async fn campaigns_router( let req = CampaignLoad.call(req, app).await?; - campaign::insert_events(req, app).await + campaign::insert_events::handle_route(req, app).await } else if let (Some(_caps), &Method::POST) = (CLOSE_CAMPAIGN_BY_CAMPAIGN_ID.captures(&path), method) { diff --git a/sentry/src/payout.rs b/sentry/src/payout.rs index 69aaefa4e..3a805e119 100644 --- a/sentry/src/payout.rs +++ b/sentry/src/payout.rs @@ -66,7 +66,7 @@ pub fn get_payout( let mut output = Output { show: true, boost: 1.0, - price: vec![(event_type.clone(), pricing.min.clone())] + price: vec![(event_type.clone(), pricing.min)] .into_iter() .collect(), }; @@ -78,7 +78,7 @@ pub fn get_payout( if output.show { let price = match output.price.get(&event_type) { Some(output_price) => { - max(pricing.min, min(pricing.max, output_price.clone())) + max(pricing.min, min(pricing.max, *output_price)) } None => max(pricing.min, pricing.max), }; diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 5936b8974..e9574f62f 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,15 +1,8 @@ -use std::collections::HashMap; - -use crate::{ - access::{self, check_access}, - success_response, Application, Auth, ResponseError, Session, -}; -use chrono::Utc; +use crate::{success_response, Application, ResponseError}; use hyper::{Body, Request, Response}; use primitives::{ adapter::Adapter, - sentry::{campaign_create::CreateCampaign, Event, SuccessResponse}, - Campaign, + sentry::{campaign_create::CreateCampaign, SuccessResponse}, }; pub async fn create_campaign( @@ -51,75 +44,413 @@ pub async fn create_campaign( Ok(success_response(serde_json::to_string(&campaign)?)) } -pub async fn insert_events( - req: Request, - app: &Application, -) -> Result, ResponseError> { - let (req_head, req_body) = req.into_parts(); +pub mod insert_events { - let auth = req_head.extensions.get::(); - let session = req_head - .extensions - .get::() - .expect("request should have session"); + use std::collections::HashMap; - let campaign = req_head - .extensions - .get::() - .expect("request should have a Campaign loaded"); + use crate::{ + access::{self, check_access}, + db::{accounting::spend_amount, DbPool, PoolError, RedisError}, + payout::get_payout, + spender::fee::calculate_fee, + Application, Auth, ResponseError, Session, + }; + use hyper::{Body, Request, Response}; + use primitives::{ + adapter::Adapter, + sentry::{ + accounting::{Balances, CheckedState, OverflowError}, + Event, SuccessResponse, + }, + Address, Campaign, CampaignId, DomainError, UnifiedNum, ValidatorDesc, + }; + use redis::aio::MultiplexedConnection; + use thiserror::Error; - let body_bytes = hyper::body::to_bytes(req_body).await?; - let mut request_body = serde_json::from_slice::>>(&body_bytes)?; + // TODO AIP#61: Use the Campaign Modify const here + pub const CAMPAIGN_REMAINING_KEY: &str = "campaignRemaining"; - let events = request_body - .remove("events") - .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; + #[derive(Debug, Error)] + pub enum Error { + #[error(transparent)] + Event(#[from] EventError), + #[error(transparent)] + Redis(#[from] RedisError), + #[error(transparent)] + Postgres(#[from] PoolError), + #[error(transparent)] + Overflow(#[from] OverflowError), + } - let processed = process_events(app, auth, session, campaign, events).await?; + #[derive(Debug, Error, PartialEq)] + pub enum EventError { + #[error("Overflow when calculating Event payout for Event")] + EventPayoutOverflow, + #[error("Validator Fee calculation: {0}")] + FeeCalculation(#[from] DomainError), + #[error( + "The Campaign's remaining budget left to spend is not enough to cover the Event payout" + )] + CampaignRemainingNotEnoughForPayout, + #[error("Campaign ran out of remaining budget to spend")] + CampaignOutOfBudget, + } - Ok(Response::builder() - .header("Content-type", "application/json") - .body(serde_json::to_string(&SuccessResponse { success: processed })?.into()) - .unwrap()) -} + pub async fn handle_route( + req: Request, + app: &Application, + ) -> Result, ResponseError> { + let (req_head, req_body) = req.into_parts(); -async fn process_events( - app: &Application, - auth: Option<&Auth>, - session: &Session, - campaign: &Campaign, - events: Vec, -) -> Result { - if &Utc::now() > &campaign.active.to { - return Err(ResponseError::BadRequest("Campaign is expired".into())); + let auth = req_head.extensions.get::(); + let session = req_head + .extensions + .get::() + .expect("request should have session"); + + let campaign = req_head + .extensions + .get::() + .expect("request should have a Campaign loaded"); + + let body_bytes = hyper::body::to_bytes(req_body).await?; + let mut request_body = serde_json::from_slice::>>(&body_bytes)?; + + let events = request_body + .remove("events") + .ok_or_else(|| ResponseError::BadRequest("invalid request".to_string()))?; + + let processed = process_events(app, auth, session, campaign, events).await?; + + Ok(Response::builder() + .header("Content-type", "application/json") + .body(serde_json::to_string(&SuccessResponse { success: processed })?.into()) + .unwrap()) + } + + async fn process_events( + app: &Application, + auth: Option<&Auth>, + session: &Session, + campaign: &Campaign, + events: Vec, + ) -> Result { + // handle events - check access + check_access( + &app.redis, + session, + auth, + &app.config.ip_rate_limit, + &campaign, + &events, + ) + .await + .map_err(|e| match e { + access::Error::ForbiddenReferrer => ResponseError::Forbidden(e.to_string()), + access::Error::RulesError(error) => ResponseError::TooManyRequests(error), + access::Error::UnAuthenticated => ResponseError::Unauthorized, + _ => ResponseError::BadRequest(e.to_string()), + })?; + + let (leader, follower) = match (campaign.leader(), campaign.follower()) { + // ERROR! + (None, None) | (None, _) | (_, None) => { + return Err(ResponseError::BadRequest( + "Channel leader, follower or both were not found in Campaign validators." + .to_string(), + )) + } + (Some(leader), Some(follower)) => (leader, follower), + }; + + let mut events_success = vec![]; + for event in events.into_iter() { + let result: Result, Error> = { + // calculate earners payouts + let payout = get_payout(&app.logger, campaign, &event, session)?; + + match payout { + Some((earner, payout)) => spend_for_event( + &app.pool, + app.redis.clone(), + &campaign, + earner, + leader, + follower, + payout, + ) + .await + .map(Some), + None => Ok(None), + } + }; + + events_success.push((event, result)); + } + + // TODO AIP#61 - aggregate Events and put into analytics + + Ok(true) + } + + pub async fn spend_for_event( + pool: &DbPool, + mut redis: MultiplexedConnection, + campaign: &Campaign, + earner: Address, + leader: &ValidatorDesc, + follower: &ValidatorDesc, + amount: UnifiedNum, + ) -> Result<(), Error> { + // distribute fees + let leader_fee = + calculate_fee((earner, amount), &leader).map_err(EventError::FeeCalculation)?; + let follower_fee = + calculate_fee((earner, amount), &follower).map_err(EventError::FeeCalculation)?; + + // First update redis `campaignRemaining:{CampaignId}` key + let spending = [amount, leader_fee, follower_fee] + .iter() + .sum::>() + .ok_or(EventError::EventPayoutOverflow)?; + + if !has_enough_remaining_budget(&mut redis, campaign.id, spending).await? { + return Err(Error::Event( + EventError::CampaignRemainingNotEnoughForPayout, + )); + } + + // The event payout decreases the remaining budget for the Campaign + let remaining = decrease_remaining_budget(&mut redis, campaign.id, spending).await?; + + // Update the Accounting records accordingly + let channel_id = campaign.channel.id(); + let spender = campaign.creator; + + let mut delta_balances = Balances::::default(); + delta_balances.spend(spender, earner, amount)?; + delta_balances.spend(spender, leader.id.to_address(), leader_fee)?; + delta_balances.spend(spender, follower.id.to_address(), follower_fee)?; + + let (_earners, _spenders) = spend_amount(pool.clone(), channel_id, delta_balances).await?; + + // check if we still have budget to spend, after we've updated both Redis and Postgres + if remaining.is_negative() { + Err(Error::Event(EventError::CampaignOutOfBudget)) + } else { + Ok(()) + } + } + + async fn has_enough_remaining_budget( + redis: &mut MultiplexedConnection, + campaign: CampaignId, + amount: UnifiedNum, + ) -> Result { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); + + let remaining = redis::cmd("GET") + .arg(&key) + .query_async::<_, Option>(redis) + .await? + .unwrap_or_default(); + + Ok(remaining > 0 && remaining.unsigned_abs() > amount.to_u64()) + } + + async fn decrease_remaining_budget( + redis: &mut MultiplexedConnection, + campaign: CampaignId, + amount: UnifiedNum, + ) -> Result { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); + + let remaining = redis::cmd("DECRBY") + .arg(&key) + .arg(amount.to_u64()) + .query_async::<_, i64>(redis) + .await?; + + Ok(remaining) } - // - // TODO #381: AIP#61 Spender Aggregator should be called - // - - // handle events - check access - // handle events - Update targeting rules - // calculate payout - // distribute fees - // handle spending - Spender Aggregate - // handle events - aggregate Events and put into analytics - - check_access( - &app.redis, - session, - auth, - &app.config.ip_rate_limit, - &campaign, - &events, - ) - .await - .map_err(|e| match e { - access::Error::ForbiddenReferrer => ResponseError::Forbidden(e.to_string()), - access::Error::RulesError(error) => ResponseError::TooManyRequests(error), - access::Error::UnAuthenticated => ResponseError::Unauthorized, - _ => ResponseError::BadRequest(e.to_string()), - })?; - - Ok(true) + #[cfg(test)] + mod test { + use primitives::util::tests::prep_db::{ADDRESSES, DUMMY_CAMPAIGN}; + + use crate::db::{ + redis_pool::TESTS_POOL, + tests_postgres::{setup_test_migrations, DATABASE_POOL}, + }; + + use super::*; + + /// Helper function to get the Campaign Remaining budget in Redis for the tests + async fn get_campaign_remaining( + redis: &mut MultiplexedConnection, + campaign: CampaignId, + ) -> Option { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); + + redis::cmd("GET") + .arg(&key) + .query_async(redis) + .await + .expect("Should set Campaign remaining key") + } + + /// Helper function to set the Campaign Remaining budget in Redis for the tests + async fn set_campaign_remaining( + redis: &mut MultiplexedConnection, + campaign: CampaignId, + remaining: i64, + ) { + let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); + + redis::cmd("SET") + .arg(&key) + .arg(remaining) + .query_async::<_, ()>(redis) + .await + .expect("Should set Campaign remaining key"); + } + + #[tokio::test] + async fn test_has_enough_remaining_budget() { + let mut redis = TESTS_POOL.get().await.expect("Should get redis connection"); + let campaign = DUMMY_CAMPAIGN.id; + let amount = UnifiedNum::from(10_000); + + let no_remaining_budget_set = has_enough_remaining_budget(&mut redis, campaign, amount) + .await + .expect("Should check campaign remaining"); + assert!( + !no_remaining_budget_set, + "No remaining budget set, should return false" + ); + + set_campaign_remaining(&mut redis, campaign, 9_000).await; + + let not_enough_remaining_budget = + has_enough_remaining_budget(&mut redis, campaign, amount) + .await + .expect("Should check campaign remaining"); + assert!( + !not_enough_remaining_budget, + "Not enough remaining budget, should return false" + ); + + set_campaign_remaining(&mut redis, campaign, 11_000).await; + + let has_enough_remaining_budget = + has_enough_remaining_budget(&mut redis, campaign, amount) + .await + .expect("Should check campaign remaining"); + + assert!( + has_enough_remaining_budget, + "Should have enough budget for this amount" + ); + } + + #[tokio::test] + async fn test_decreasing_remaining_budget() { + let mut redis = TESTS_POOL.get().await.expect("Should get redis connection"); + let campaign = DUMMY_CAMPAIGN.id; + let amount = UnifiedNum::from(5_000); + + set_campaign_remaining(&mut redis, campaign, 9_000).await; + + let remaining = decrease_remaining_budget(&mut redis, campaign, amount) + .await + .expect("Should decrease campaign remaining"); + assert_eq!( + 4_000, remaining, + "Should decrease remaining budget with amount and be positive" + ); + + let remaining = decrease_remaining_budget(&mut redis, campaign, amount) + .await + .expect("Should decrease campaign remaining"); + assert_eq!( + -1_000, remaining, + "Should decrease remaining budget with amount and be negative" + ); + } + + #[tokio::test] + async fn test_spending_for_events_with_enough_remaining_budget() { + let mut redis = TESTS_POOL.get().await.expect("Should get redis connection"); + let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); + + setup_test_migrations(database.pool.clone()) + .await + .expect("Migrations should succeed"); + + let campaign = DUMMY_CAMPAIGN.clone(); + + let publisher = ADDRESSES["publisher"]; + + let leader = campaign.leader().unwrap(); + let follower = campaign.follower().unwrap(); + let payout = UnifiedNum::from(300); + + // No Campaign Remaining set, should error + { + let spend_event = spend_for_event( + &database.pool, + redis.connection.clone(), + &campaign, + publisher, + leader, + follower, + payout, + ) + .await; + + assert!( + matches!( + spend_event, + Err(Error::Event( + EventError::CampaignRemainingNotEnoughForPayout + )) + ), + "Campaign budget has no remaining funds to spend" + ); + } + + // Repeat the same call, but set the Campaign remaining budget in Redis + { + set_campaign_remaining(&mut redis, campaign.id, 11_000).await; + + let spend_event = spend_for_event( + &database.pool, + redis.connection.clone(), + &campaign, + publisher, + leader, + follower, + payout, + ) + .await; + + assert!( + dbg!(spend_event).is_ok(), + "Campaign budget has no remaining funds to spend" + ); + + // Payout: 300 + // Leader fee: 100 + // Leader payout: 300 * 100 / 1000 = 30 + // Follower fee: 100 + // Follower payout: 300 * 100 / 1000 = 30 + assert_eq!( + 10_640_i64, + get_campaign_remaining(&mut redis.connection, campaign.id) + .await + .expect("Should have key") + ) + } + } + } } diff --git a/sentry/src/spender.rs b/sentry/src/spender.rs index da6b9cacd..46f91bc7e 100644 --- a/sentry/src/spender.rs +++ b/sentry/src/spender.rs @@ -26,31 +26,19 @@ impl Aggregator { pub mod fee { pub const PRO_MILLE: UnifiedNum = UnifiedNum::from_u64(1_000); - use primitives::{Address, Campaign, DomainError, UnifiedNum, ValidatorId}; + use primitives::{Address, DomainError, UnifiedNum, ValidatorDesc}; /// Calculates the fee for a specified validator /// This function will return None if the provided validator is not part of the Campaign / Channel /// In the case of overflow when calculating the payout, an error will be returned - pub fn calculate_fees( + pub fn calculate_fee( (_earner, payout): (Address, UnifiedNum), - campaign: &Campaign, - for_validator: ValidatorId, - ) -> Result, DomainError> { - let payout = match campaign.find_validator(&for_validator) { - Some(validator_role) => { - // should never overflow - let fee_payout = payout - .checked_mul(&validator_role.validator().fee) - .ok_or_else(|| { - DomainError::InvalidArgument("payout calculation overflow".to_string()) - })? - .div_floor(&PRO_MILLE); - - Some(fee_payout) - } - None => None, - }; - - Ok(payout) + validator: &ValidatorDesc, + ) -> Result { + // should never overflow + payout + .checked_mul(&validator.fee) + .map(|pro_mille_fee| pro_mille_fee.div_floor(&PRO_MILLE)) + .ok_or_else(|| DomainError::InvalidArgument("payout calculation overflow".to_string())) } } From 8fcef1313747034bdb867167d6b11b6008a9ba9c Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 13 Jul 2021 14:02:27 +0300 Subject: [PATCH 33/49] sentry - routes - modify campaign --- primitives/src/sentry.rs | 45 ++-- sentry/src/db/campaign.rs | 131 +++++------ sentry/src/db/spendable.rs | 8 +- sentry/src/lib.rs | 4 +- sentry/src/routes/campaign.rs | 432 +++++++++++++++++++++------------- 5 files changed, 368 insertions(+), 252 deletions(-) diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index b9c9c8f0c..6018dc235 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -386,22 +386,37 @@ pub mod campaign_create { } } - pub fn apply(&self, campaign: &Campaign) -> Campaign { - let campaign = campaign.clone(); - Campaign { - id: campaign.id, - channel: campaign.channel, - creator: campaign.creator, - budget: self.budget.unwrap_or(campaign.budget), - validators: self.validators.clone().unwrap_or(campaign.validators), - title: self.title.clone().or(campaign.title), - pricing_bounds: self.pricing_bounds.clone().or(campaign.pricing_bounds), - event_submission: self.event_submission.clone().or(campaign.event_submission), - ad_units: self.ad_units.clone().unwrap_or(campaign.ad_units), - targeting_rules: self.targeting_rules.clone().unwrap_or(campaign.targeting_rules), - created: campaign.created, - active: campaign.active, + pub fn apply(self, mut campaign: Campaign) -> Campaign { + if let Some(new_budget) = self.budget { + campaign.budget = new_budget; + } + + if let Some(new_validators) = self.validators { + campaign.validators = new_validators; + } + + // check if it was passed otherwise not sending a Title will result in clearing of the current one + if let Some(new_title) = self.title { + campaign.title = Some(new_title); + } + + if let Some(new_pricing_bounds) = self.pricing_bounds { + campaign.pricing_bounds = Some(new_pricing_bounds); } + + if let Some(new_event_submission) = self.event_submission { + campaign.event_submission = Some(new_event_submission); + } + + if let Some(new_ad_units) = self.ad_units { + campaign.ad_units = new_ad_units; + } + + if let Some(new_targeting_rules) = self.targeting_rules { + campaign.targeting_rules = new_targeting_rules; + } + + campaign } } } diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 75989a420..0c00b78c9 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -63,20 +63,21 @@ pub async fn get_campaigns_by_channel( Ok(campaigns) } -pub async fn update_campaign( - pool: &DbPool, - campaign: &Campaign, -) -> Result { +/// ```text +/// UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 +/// WHERE id = $8 +/// RETURNING id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to +/// +pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let statement = client - .prepare("UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 WHERE id = $8") + .prepare("UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 WHERE id = $8 RETURNING id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to") .await?; - let ad_units = Json(&campaign.ad_units); - let updated_rows = client - .execute( + let updated_row = client + .query_one( &statement, &[ &campaign.budget, @@ -91,106 +92,100 @@ pub async fn update_campaign( ) .await?; - let exists = updated_rows == 1; - Ok(exists) + Ok(Campaign::from(&updated_row)) } #[cfg(test)] mod test { use primitives::{ - util::tests::prep_db::{DUMMY_CAMPAIGN, DUMMY_AD_UNITS}, - event_submission::{Rule, RateLimit}, + campaign, + event_submission::{RateLimit, Rule}, sentry::campaign_create::ModifyCampaign, targeting::Rules, - UnifiedNum, EventSubmission, + util::tests::prep_db::{DUMMY_AD_UNITS, DUMMY_CAMPAIGN}, + EventSubmission, UnifiedNum, }; - use primitives::campaign; use std::time::Duration; use tokio_postgres::error::SqlState; - use crate::{ - db::tests_postgres::{setup_test_migrations, DATABASE_POOL}, - }; + use crate::db::tests_postgres::{setup_test_migrations, DATABASE_POOL}; use super::*; #[tokio::test] - async fn it_inserts_and_fetches_campaign() { + async fn it_inserts_fetches_and_updates_a_campaign() { let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); setup_test_migrations(database.pool.clone()) .await .expect("Migrations should succeed"); - let campaign_for_testing = DUMMY_CAMPAIGN.clone(); + let campaign = DUMMY_CAMPAIGN.clone(); - let non_existent_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing.id) + let non_existent_campaign = fetch_campaign(database.pool.clone(), &campaign.id) .await .expect("Should fetch successfully"); assert_eq!(None, non_existent_campaign); - let is_inserted = insert_campaign(&database.pool, &campaign_for_testing) + let is_inserted = insert_campaign(&database.pool, &campaign) .await .expect("Should succeed"); assert!(is_inserted); - let is_duplicate_inserted = insert_campaign(&database.pool, &campaign_for_testing).await; + let is_duplicate_inserted = insert_campaign(&database.pool, &campaign).await; assert!(matches!( is_duplicate_inserted, Err(PoolError::Backend(error)) if error.code() == Some(&SqlState::UNIQUE_VIOLATION) )); - let fetched_campaign = fetch_campaign(database.pool.clone(), &campaign_for_testing.id) + let fetched_campaign = fetch_campaign(database.pool.clone(), &campaign.id) .await .expect("Should fetch successfully"); - assert_eq!(Some(campaign_for_testing), fetched_campaign); - } - - #[tokio::test] - async fn it_updates_campaign() { - let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); - - setup_test_migrations(database.pool.clone()) - .await - .expect("Migrations should succeed"); - - let campaign_for_testing = DUMMY_CAMPAIGN.clone(); - - let is_inserted = insert_campaign(&database.pool, &campaign_for_testing) - .await - .expect("Should succeed"); - - assert!(is_inserted); - - let rule = Rule { - uids: None, - rate_limit: Some(RateLimit { - limit_type: "sid".to_string(), - time_frame: Duration::from_millis(20_000), - }), - }; - let new_budget = campaign_for_testing.budget + UnifiedNum::from_u64(1_000_000_000); - let modified_campaign = ModifyCampaign { - // pub budget: Option, - budget: Some(new_budget), - validators: None, - title: Some("Modified Campaign".to_string()), - pricing_bounds: Some(campaign::PricingBounds { - impression: Some(campaign::Pricing { min: 1.into(), max: 10.into()}), - click: Some(campaign::Pricing { min: 0.into(), max: 0.into()}) - }), - event_submission: Some(EventSubmission { allow: vec![rule] }), - ad_units: Some(DUMMY_AD_UNITS.to_vec()), - targeting_rules: Some(Rules::new()), - }; - - let applied_campaign = modified_campaign.apply(&campaign_for_testing); - - let is_campaign_updated = update_campaign(&database.pool, &applied_campaign).await.expect("should update"); - assert!(is_campaign_updated); + assert_eq!(Some(campaign.clone()), fetched_campaign); + + // Update campaign + { + let rule = Rule { + uids: None, + rate_limit: Some(RateLimit { + limit_type: "sid".to_string(), + time_frame: Duration::from_millis(20_000), + }), + }; + let new_budget = campaign.budget + UnifiedNum::from_u64(1_000_000_000); + let modified_campaign = ModifyCampaign { + budget: Some(new_budget), + validators: None, + title: Some("Modified Campaign".to_string()), + pricing_bounds: Some(campaign::PricingBounds { + impression: Some(campaign::Pricing { + min: 1.into(), + max: 10.into(), + }), + click: Some(campaign::Pricing { + min: 0.into(), + max: 0.into(), + }), + }), + event_submission: Some(EventSubmission { allow: vec![rule] }), + ad_units: Some(DUMMY_AD_UNITS.to_vec()), + targeting_rules: Some(Rules::new()), + }; + + let applied_campaign = modified_campaign.apply(campaign.clone()); + + let updated_campaign = update_campaign(&database.pool, &applied_campaign) + .await + .expect("should update"); + + assert_eq!( + applied_campaign, updated_campaign, + "Postgres should update all modified fields" + ); + } } } diff --git a/sentry/src/db/spendable.rs b/sentry/src/db/spendable.rs index 67dc8fcb2..a6c340c2f 100644 --- a/sentry/src/db/spendable.rs +++ b/sentry/src/db/spendable.rs @@ -37,13 +37,13 @@ pub async fn fetch_spendable( pool: DbPool, spender: &Address, channel_id: &ChannelId, -) -> Result { +) -> Result, PoolError> { let client = pool.get().await?; let statement = client.prepare("SELECT spender, channel_id, channel, total, still_on_create2 FROM spendable WHERE spender = $1 AND channel_id = $2").await?; - let row = client.query_one(&statement, &[spender, channel_id]).await?; + let row = client.query_opt(&statement, &[spender, channel_id]).await?; - Ok(Spendable::try_from(row)?) + Ok(row.map(Spendable::try_from).transpose()?) } #[cfg(test)] @@ -88,6 +88,6 @@ mod test { .await .expect("Should fetch successfully"); - assert_eq!(spendable, fetched_spendable); + assert_eq!(Some(spendable), fetched_spendable); } } diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index e16be4ac3..c5ff2ef83 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::all)] #![deny(rust_2018_idioms)] -use crate::db::{DbPool}; +use crate::db::DbPool; use crate::routes::campaign; use crate::routes::channel::channel_status; use crate::routes::event_aggregate::list_channel_event_aggregates; @@ -437,7 +437,7 @@ pub fn bad_response(response_body: String, status_code: StatusCode) -> Response< let mut error_response = HashMap::new(); error_response.insert("message", response_body); - let body = Body::from(serde_json::to_string(&error_response).expect("serialise err response")); + let body = Body::from(serde_json::to_string(&error_response).expect("serialize err response")); let mut response = Response::new(body); response diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index e37502c91..f7612dd04 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,41 +1,34 @@ use crate::{ - success_response, Application, ResponseError, + access::{self, check_access}, db::{ - spendable::fetch_spendable, accounting::get_accounting_spent, - campaign::{update_campaign, insert_campaign, get_campaigns_by_channel}, - DbPool + campaign::{get_campaigns_by_channel, insert_campaign, update_campaign}, + spendable::fetch_spendable, + DbPool, }, routes::campaign::update_campaign::set_initial_remaining_for_campaign, - access::{self, check_access}, - Auth, Session + success_response, Application, Auth, ResponseError, Session, }; - +use chrono::Utc; +use deadpool_postgres::PoolError; use hyper::{Body, Request, Response}; use primitives::{ adapter::Adapter, + campaign_validator::Validator, sentry::{ campaign_create::{CreateCampaign, ModifyCampaign}, Event, SuccessResponse, }, - campaign_validator::Validator, - Campaign, CampaignId, UnifiedNum -}; -use redis::{ - aio::MultiplexedConnection, - RedisError, + Address, Campaign, CampaignId, UnifiedNum, }; +use redis::{aio::MultiplexedConnection, RedisError}; use slog::error; use std::{ cmp::{max, Ordering}, collections::HashMap, - convert::TryInto, }; -use deadpool_postgres::PoolError; -use tokio_postgres::error::SqlState; - -use chrono::Utc; use thiserror::Error; +use tokio_postgres::error::SqlState; #[derive(Debug, Error)] pub enum Error { @@ -47,10 +40,14 @@ pub enum Error { BudgetExceeded, #[error("Error with new budget: {0}")] NewBudget(String), + #[error("Spendable amount for campaign creator {0} not found")] + SpenderNotFound(Address), + #[error("Campaign was not modified because of spending constraints")] + CampaignNotModified, #[error("Redis error: {0}")] Redis(#[from] RedisError), #[error("DB Pool error: {0}")] - Pool(#[from] PoolError) + Pool(#[from] PoolError), } pub async fn create_campaign( @@ -70,28 +67,51 @@ pub async fn create_campaign( // create the actual `Campaign` with random `CampaignId` .into_campaign(); - campaign.validate(&app.config, &app.adapter.whoami()).map_err(|_| ResponseError::FailedValidation("couldn't valdiate campaign".to_string()))?; + campaign + .validate(&app.config, &app.adapter.whoami()) + .map_err(|_| ResponseError::FailedValidation("couldn't valdiate campaign".to_string()))?; if auth.uid.to_address() != campaign.creator { - return Err(ResponseError::Forbidden("Request not sent by campaign creator".to_string())) + return Err(ResponseError::Forbidden( + "Request not sent by campaign creator".to_string(), + )); } - let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); + let error_response = + ResponseError::BadRequest("err occurred; please try again later".to_string()); - let accounting_spent = get_accounting_spent(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + let accounting_spent = + get_accounting_spent(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; - // TODO: AIP#61: Update when changes to Spendable are ready - let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + let latest_spendable = + fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()) + .await? + .ok_or(ResponseError::BadRequest( + "No spendable amount found for the Campaign creator".to_string(), + ))?; let total_deposited = latest_spendable.deposit.total; - let remaining_for_channel = total_deposited.checked_sub(&accounting_spent).ok_or(ResponseError::FailedValidation("No more budget remaining".to_string()))?; + let remaining_for_channel = + total_deposited + .checked_sub(&accounting_spent) + .ok_or(ResponseError::FailedValidation( + "No more budget remaining".to_string(), + ))?; if campaign.budget > remaining_for_channel { - return Err(ResponseError::BadRequest("Not enough deposit left for the new campaign budget".to_string())); + return Err(ResponseError::BadRequest( + "Not enough deposit left for the new campaign budget".to_string(), + )); } // If the campaign is being created, the amount spent is 0, therefore remaining = budget - set_initial_remaining_for_campaign(&mut app.redis.clone(), campaign.id, campaign.budget).await.map_err(|_| ResponseError::BadRequest("Couldn't update remaining while creating campaign".to_string()))?; + set_initial_remaining_for_campaign(&app.redis, campaign.id, campaign.budget) + .await + .map_err(|_| { + ResponseError::BadRequest( + "Couldn't update remaining while creating campaign".to_string(), + ) + })?; // insert Campaign match insert_campaign(&app.pool, &campaign).await { @@ -106,7 +126,9 @@ pub async fn create_campaign( _ => Err(error_response), } } - Ok(false) => Err(ResponseError::BadRequest("Encountered error while creating Campaign; please try again".to_string())), + Ok(false) => Err(ResponseError::BadRequest( + "Encountered error while creating Campaign; please try again".to_string(), + )), _ => Ok(()), }?; @@ -118,167 +140,266 @@ pub mod update_campaign { pub const CAMPAIGN_REMAINING_KEY: &'static str = "campaignRemaining"; - pub async fn set_initial_remaining_for_campaign(redis: &mut MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + pub async fn set_initial_remaining_for_campaign( + redis: &MultiplexedConnection, + id: CampaignId, + amount: UnifiedNum, + ) -> Result { let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); redis::cmd("SETNX") .arg(&key) .arg(amount.to_u64()) - .query_async(redis) + .query_async(&mut redis.clone()) .await?; Ok(true) } - pub async fn increase_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Result { + pub async fn increase_remaining_for_campaign( + redis: &MultiplexedConnection, + id: CampaignId, + amount: UnifiedNum, + ) -> Result { let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); redis::cmd("INCRBY") .arg(&key) .arg(amount.to_u64()) - .query_async(&mut redis.clone()) - .await?; - Ok(true) + .query_async::<_, u64>(&mut redis.clone()) + .await + .map(UnifiedNum::from) } - pub async fn decrease_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId, amount: UnifiedNum) -> Option { + pub async fn decrease_remaining_for_campaign( + redis: &MultiplexedConnection, + id: CampaignId, + amount: UnifiedNum, + ) -> Result { let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); - let value = match redis::cmd("DECRBY") + redis::cmd("DECRBY") .arg(&key) .arg(amount.to_u64()) - .query_async::<_, Option>(&mut redis.clone()) + .query_async::<_, i64>(&mut redis.clone()) .await - { - Ok(Some(remaining)) => { - // Can't be less than 0 due to max() - Some(UnifiedNum::from_u64(max(0, remaining).try_into().unwrap())) - }, - _ => None - }; - value } pub async fn handle_route( req: Request, app: &Application, ) -> Result, ResponseError> { - let campaign_being_mutated = req.extensions().get::().expect("We must have a campaign in extensions").to_owned(); + let campaign_being_mutated = req + .extensions() + .get::() + .expect("We must have a campaign in extensions") + .to_owned(); let body = hyper::body::to_bytes(req.into_body()).await?; - let modified_campaign = serde_json::from_slice::(&body) + let modify_campaign_fields = serde_json::from_slice::(&body) .map_err(|e| ResponseError::FailedValidation(e.to_string()))?; - let modified_campaign = ModifyCampaign::from_campaign(modified_campaign.clone()); - // modify Campaign - modify_campaign(&app.pool, &campaign_being_mutated, &modified_campaign, &app.redis).await.map_err(|_| ResponseError::BadRequest("Failed to update campaign".to_string()))?; + let modified_campaign = modify_campaign( + &app.pool, + &mut app.redis.clone(), + campaign_being_mutated, + modify_campaign_fields, + ) + .await + .map_err(|err| ResponseError::BadRequest(err.to_string()))?; Ok(success_response(serde_json::to_string(&modified_campaign)?)) } - pub async fn modify_campaign(pool: &DbPool, campaign: &Campaign, modified_campaign: &ModifyCampaign, redis: &MultiplexedConnection) -> Result { + pub async fn modify_campaign( + pool: &DbPool, + redis: &MultiplexedConnection, + campaign: Campaign, + modify_campaign: ModifyCampaign, + ) -> Result { // *NOTE*: When updating campaigns make sure sum(campaigns.map(getRemaining)) <= totalDepoisted - totalspent // !WARNING!: totalSpent != sum(campaign.map(c => c.spending)) therefore we must always calculate remaining funds based on total_deposit - lastApprovedNewState.spenders[user] // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 - if let Some(new_budget) = modified_campaign.budget { - let old_remaining = get_remaining_for_campaign(&redis, campaign.id).await?.ok_or(Error::FailedUpdate("No remaining entry for campaign".to_string()))?; + let delta_budget = if let Some(new_budget) = modify_campaign.budget { + get_delta_budget(redis, &campaign, new_budget).await? + } else { + None + }; + + // if we are going to update the budget + // validate the totalDeposit - totalSpent for all campaign + // sum(AllChannelCampaigns.map(getRemaining)) + DeltaBudgetForMutatedCampaign <= totalDeposited - totalSpent + // sum(AllChannelCampaigns.map(getRemaining)) - DeltaBudgetForMutatedCampaign <= totalDeposited - totalSpent + if let Some(delta_budget) = delta_budget { + let accounting_spent = + get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()) + .await?; + + let latest_spendable = + fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()) + .await? + .ok_or(Error::SpenderNotFound(campaign.creator))?; + + // Gets the latest Spendable for this (spender, channelId) pair + let total_deposited = latest_spendable.deposit.total; + + let total_remaining = total_deposited + .checked_sub(&accounting_spent) + .ok_or(Error::Calculation)?; + let channel_campaigns = get_campaigns_by_channel(&pool, &campaign.channel.id()).await?; + + // this will include the Campaign we are currently modifying + let campaigns_current_remaining_sum = + get_remaining_for_multiple_campaigns(&redis, &channel_campaigns) + .await? + .iter() + .sum::>() + .ok_or(Error::Calculation)?; + + // apply the delta_budget to the sum + let new_campaigns_remaining = match delta_budget { + DeltaBudget::Increase(increase_by) => { + campaigns_current_remaining_sum.checked_add(&increase_by) + } + DeltaBudget::Decrease(decrease_by) => { + campaigns_current_remaining_sum.checked_sub(&decrease_by) + } + } + .ok_or(Error::Calculation)?; - let campaign_spent = campaign.budget.checked_sub(&old_remaining).ok_or(Error::Calculation)?; - if campaign_spent >= new_budget { - return Err(Error::NewBudget("New budget should be greater than the spent amount".to_string())); + if !(new_campaigns_remaining <= total_remaining) { + return Err(Error::CampaignNotModified); } - // Separate variable for clarity - let old_budget = campaign.budget; - - match new_budget.cmp(&old_budget) { - Ordering::Equal => (), - Ordering::Greater => { - let new_remaining = old_remaining.checked_add(&new_budget.checked_sub(&old_budget).ok_or(Error::Calculation)?).ok_or(Error::Calculation)?; - let amount_to_incr = new_remaining.checked_sub(&old_remaining).ok_or(Error::Calculation)?; - increase_remaining_for_campaign(&redis, campaign.id, amount_to_incr).await?; - }, - Ordering::Less => { - let new_remaining = old_remaining.checked_add(&old_budget.checked_sub(&new_budget).ok_or(Error::Calculation)?).ok_or(Error::Calculation)?; - let amount_to_decr = new_remaining.checked_sub(&old_remaining).ok_or(Error::Calculation)?; - let decreased_remaining = decrease_remaining_for_campaign(&redis, campaign.id, amount_to_decr).await.ok_or(Error::FailedUpdate("Could't decrease remaining".to_string()))?; - // If it goes below 0 it will still return 0 - if decreased_remaining.eq(&UnifiedNum::from_u64(0)) { - return Err(Error::NewBudget("No budget remaining after decreasing".to_string())); + // if the value is not positive it will return an error because of UnifiedNum + let _campaign_remaining = match delta_budget { + // should always be positive + DeltaBudget::Increase(increase_by) => { + increase_remaining_for_campaign(redis, campaign.id, increase_by).await? + } + // there is a chance that an even lowered the remaining and it's no longer positive + // check if positive and create an UnifiedNum, or return an error + DeltaBudget::Decrease(decrease_by) => { + match decrease_remaining_for_campaign(redis, campaign.id, decrease_by).await? { + remaining if remaining >= 0 => UnifiedNum::from(remaining.unsigned_abs()), + _ => UnifiedNum::from(0), } } - } - }; + }; + } - let accounting_spent = get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + let modified_campaign = modify_campaign.apply(campaign); + update_campaign(&pool, &modified_campaign).await?; - let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + Ok(modified_campaign) + } - // Gets the latest Spendable for this (spender, channelId) pair - let total_deposited = latest_spendable.deposit.total; + /// Delta Budget describes the difference between the New and Old budget + /// It is used to decrease or increase the remaining budget instead of setting it up directly + /// This way if a new event alters the remaining budget in Redis while the modification of campaign hasn't finished + /// it will correctly update the remaining using an atomic redis operation with `INCRBY` or `DECRBY` instead of using `SET` + enum DeltaBudget { + Increase(T), + Decrease(T), + } - let total_remaining = total_deposited.checked_sub(&accounting_spent).ok_or(Error::Calculation)?; - let campaigns_for_channel = get_campaigns_by_channel(&pool, &campaign.channel.id()).await?; - let current_campaign_budget = modified_campaign.budget.unwrap_or(campaign.budget); - let campaigns_remaining_sum = get_campaigns_remaining_sum(&redis, &campaigns_for_channel, campaign.id, ¤t_campaign_budget).await.map_err(|_| Error::Calculation)?; - if campaigns_remaining_sum <= total_remaining { - let campaign_with_updates = modified_campaign.apply(campaign); - update_campaign(&pool, &campaign_with_updates).await?; + async fn get_delta_budget( + redis: &MultiplexedConnection, + campaign: &Campaign, + new_budget: UnifiedNum, + ) -> Result>, Error> { + let current_budget = campaign.budget; + + let budget_action = match new_budget.cmp(¤t_budget) { + // if there is no difference in budgets - no action needed + Ordering::Equal => return Ok(None), + Ordering::Greater => DeltaBudget::Increase(()), + Ordering::Less => DeltaBudget::Decrease(()), + }; + + let old_remaining = get_remaining_for_campaign(redis, campaign.id) + .await? + .ok_or(Error::FailedUpdate( + "No remaining entry for campaign".to_string(), + ))?; + + let campaign_spent = campaign + .budget + .checked_sub(&old_remaining) + .ok_or(Error::Calculation)?; + + if campaign_spent >= new_budget { + return Err(Error::NewBudget( + "New budget should be greater than the spent amount".to_string(), + )); } - Ok(campaign.clone()) + let budget = match budget_action { + DeltaBudget::Increase(()) => { + // delta budget = New budget - Old budget ( the difference between the new and old when New > Old) + let new_remaining = new_budget + .checked_sub(¤t_budget) + .and_then(|delta_budget| old_remaining.checked_add(&delta_budget)) + .ok_or(Error::Calculation)?; + let increase_by = new_remaining + .checked_sub(&old_remaining) + .ok_or(Error::Calculation)?; + + DeltaBudget::Increase(increase_by) + } + DeltaBudget::Decrease(()) => { + // delta budget = New budget - Old budget ( the difference between the new and old when New > Old) + let new_remaining = ¤t_budget + .checked_sub(&new_budget) + .and_then(|delta_budget| old_remaining.checked_add(&delta_budget)) + .ok_or(Error::Calculation)?; + let decrease_by = new_remaining + .checked_sub(&old_remaining) + .ok_or(Error::Calculation)?; + + DeltaBudget::Decrease(decrease_by) + } + }; + + Ok(Some(budget)) } - pub async fn get_remaining_for_campaign(redis: &MultiplexedConnection, id: CampaignId) -> Result, RedisError> { + pub async fn get_remaining_for_campaign( + redis: &MultiplexedConnection, + id: CampaignId, + ) -> Result, RedisError> { let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); - let remaining = match redis::cmd("GET") + + let remaining = redis::cmd("GET") .arg(&key) .query_async::<_, Option>(&mut redis.clone()) - .await { - Ok(Some(remaining)) => { - // Can't be negative due to max() - Some(UnifiedNum::from_u64(max(0, remaining).try_into().unwrap())) - }, - Ok(None) => None, - Err(e) => return Err(e), - }; + .await? + .map(|remaining| UnifiedNum::from(max(0, remaining).unsigned_abs())); + Ok(remaining) } - async fn get_remaining_for_multiple_campaigns(redis: &MultiplexedConnection, campaigns: &[Campaign]) -> Result, Error> { - let keys: Vec = campaigns.iter().map(|c| format!("{}:{}", CAMPAIGN_REMAINING_KEY, c.id)).collect(); + async fn get_remaining_for_multiple_campaigns( + redis: &MultiplexedConnection, + campaigns: &[Campaign], + ) -> Result, Error> { + let keys: Vec = campaigns + .iter() + .map(|c| format!("{}:{}", CAMPAIGN_REMAINING_KEY, c.id)) + .collect(); + let remainings = redis::cmd("MGET") .arg(keys) .query_async::<_, Vec>>(&mut redis.clone()) - .await?; - - let remainings = remainings + .await? .into_iter() - .map(|r| { - match r { - // Can't be negative due to max() - Some(remaining) => UnifiedNum::from_u64(max(0, remaining).try_into().unwrap()), - None => UnifiedNum::from_u64(0) - } + .map(|remaining| match remaining { + Some(remaining) => UnifiedNum::from_u64(max(0, remaining).unsigned_abs()), + None => UnifiedNum::from_u64(0), }) .collect(); Ok(remainings) } - - pub async fn get_campaigns_remaining_sum(redis: &MultiplexedConnection, campaigns: &[Campaign], mutated_campaign: CampaignId, new_budget: &UnifiedNum) -> Result { - let other_campaigns_remaining = get_remaining_for_multiple_campaigns(&redis, &campaigns).await?; - let sum_of_campaigns_remaining = other_campaigns_remaining - .iter() - .sum::>() - .ok_or(Error::Calculation)?; - - // Necessary to do it explicitly for current campaign as its budget is not yet updated in DB - let old_remaining_for_mutated_campaign = get_remaining_for_campaign(&redis, mutated_campaign).await?.ok_or(Error::FailedUpdate("No remaining entry for campaign".to_string()))?; - let spent_for_mutated_campaign = new_budget.checked_sub(&old_remaining_for_mutated_campaign).ok_or(Error::Calculation)?; - let new_remaining_for_mutated_campaign = new_budget.checked_sub(&spent_for_mutated_campaign).ok_or(Error::Calculation)?; - sum_of_campaigns_remaining.checked_add(&new_remaining_for_mutated_campaign).ok_or(Error::Calculation)?; - Ok(sum_of_campaigns_remaining) - } } pub async fn insert_events( @@ -356,27 +477,12 @@ async fn process_events( #[cfg(test)] mod test { - use primitives::{ - util::tests::prep_db::{DUMMY_CAMPAIGN}, - spender::{Deposit, Spendable}, - Address - }; + use super::*; use crate::{ + campaign::update_campaign::{increase_remaining_for_campaign, CAMPAIGN_REMAINING_KEY}, db::redis_pool::TESTS_POOL, - campaign::update_campaign::{CAMPAIGN_REMAINING_KEY, increase_remaining_for_campaign}, }; - use super::*; - - // fn get_dummy_spendable(spender: Address, campaign: Campaign) -> Spendable { - // Spendable { - // spender, - // channel: campaign.channel.clone(), - // deposit: Deposit { - // total: UnifiedNum::from_u64(1_000_000), - // still_on_create2: UnifiedNum::from_u64(0), - // }, - // } - // } + use primitives::util::tests::prep_db::DUMMY_CAMPAIGN; #[tokio::test] async fn does_it_increase_remaining() { @@ -387,29 +493,32 @@ mod test { // Setting the redis base variable redis::cmd("SET") .arg(&key) - .arg(100u64) + .arg(100_u64) .query_async::<_, ()>(&mut redis.connection) .await .expect("should set"); // 2 async calls at once, should be 500 after them - futures::future::join( + futures::future::try_join_all([ + increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)), increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)), - increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)) - ).await; + ]) + .await + .expect("Should increase remaining twice"); let remaining = redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.connection) + .query_async::<_, Option>(&mut redis.connection) .await .expect("should get remaining"); - assert_eq!(remaining.is_some(), true); - - let remaining = remaining.expect("should get remaining"); - let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); - assert_eq!(remaining, UnifiedNum::from_u64(500)); + assert_eq!( + remaining.map(UnifiedNum::from_u64), + Some(UnifiedNum::from_u64(500)) + ); - increase_remaining_for_campaign(&redis, campaign.id, campaign.budget).await.expect("should increase"); + increase_remaining_for_campaign(&redis, campaign.id, campaign.budget) + .await + .expect("should increase"); let remaining = redis::cmd("GET") .arg(&key) @@ -417,23 +526,20 @@ mod test { .query_async::<_, Option>(&mut redis.connection) .await .expect("should get remaining"); - assert_eq!(remaining.is_some(), true); - let remaining = remaining.expect("should get result out of the option"); let should_be_remaining = UnifiedNum::from_u64(500) + campaign.budget; - assert_eq!(UnifiedNum::from_u64(remaining), should_be_remaining); + assert_eq!(remaining.map(UnifiedNum::from), Some(should_be_remaining)); - increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(0)).await.expect("should work"); + increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(0)) + .await + .expect("should increase remaining"); let remaining = redis::cmd("GET") .arg(&key) - .query_async::<_, Option>(&mut redis.connection) + .query_async::<_, Option>(&mut redis.connection) .await .expect("should get remaining"); - assert_eq!(remaining.is_some(), true); - let remaining = remaining.expect("should get remaining"); - let remaining = UnifiedNum::from_u64(remaining.parse::().expect("should parse")); - assert_eq!(remaining, should_be_remaining); + assert_eq!(remaining.map(UnifiedNum::from), Some(should_be_remaining)); } -} \ No newline at end of file +} From b664dd1924894ae32191e7f978a6aa9912e0ac7c Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 13 Jul 2021 14:04:24 +0300 Subject: [PATCH 34/49] sentry - db - campaign - fix doc comment --- sentry/src/db/campaign.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 0c00b78c9..fed600754 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -67,7 +67,7 @@ pub async fn get_campaigns_by_channel( /// UPDATE campaigns SET budget = $1, validators = $2, title = $3, pricing_bounds = $4, event_submission = $5, ad_units = $6, targeting_rules = $7 /// WHERE id = $8 /// RETURNING id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to -/// +/// ``` pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let statement = client From 10deba9789a5bc324ca30222c490b0233e4850ea Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 4 Aug 2021 17:35:09 +0300 Subject: [PATCH 35/49] sentry - mod test_util - setup_dummy_app helper fn --- sentry/src/lib.rs | 47 +++++++++++++++++++++++++++++++ sentry/src/middleware/campaign.rs | 45 ++--------------------------- 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index c5ff2ef83..ca5bde526 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -500,3 +500,50 @@ pub struct Auth { pub era: i64, pub uid: ValidatorId, } + +#[cfg(test)] +pub mod test_util { + use adapter::DummyAdapter; + use primitives::{ + adapter::DummyAdapterOptions, + config::configuration, + util::tests::{ + discard_logger, + prep_db::{IDS}, + }, + }; + + use crate::{Application, db::{ + redis_pool::TESTS_POOL, + tests_postgres::{setup_test_migrations, DATABASE_POOL}, + }}; + + pub async fn setup_dummy_app() -> Application { + let config = configuration("development", None).expect("Should get Config"); + let adapter = DummyAdapter::init( + DummyAdapterOptions { + dummy_identity: IDS["leader"], + dummy_auth: Default::default(), + dummy_auth_tokens: Default::default(), + }, + &config, + ); + + let redis = TESTS_POOL.get().await.expect("Should return Object"); + let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); + + setup_test_migrations(database.pool.clone()) + .await + .expect("Migrations should succeed"); + + let app = Application::new( + adapter, + config, + discard_logger(), + redis.connection.clone(), + database.pool.clone(), + ); + + app + } +} diff --git a/sentry/src/middleware/campaign.rs b/sentry/src/middleware/campaign.rs index 42d16b281..024dc1400 100644 --- a/sentry/src/middleware/campaign.rs +++ b/sentry/src/middleware/campaign.rs @@ -37,57 +37,18 @@ impl Middleware for CampaignLoad { #[cfg(test)] mod test { - use adapter::DummyAdapter; use primitives::{ - adapter::DummyAdapterOptions, - config::configuration, - util::tests::{ - discard_logger, - prep_db::{DUMMY_CAMPAIGN, IDS}, - }, + util::tests::prep_db::{DUMMY_CAMPAIGN, IDS}, Campaign, }; - use crate::db::{ - insert_campaign, - redis_pool::TESTS_POOL, - tests_postgres::{setup_test_migrations, DATABASE_POOL}, - }; + use crate::{db::insert_campaign, test_util::setup_dummy_app}; use super::*; - async fn setup_app() -> Application { - let config = configuration("development", None).expect("Should get Config"); - let adapter = DummyAdapter::init( - DummyAdapterOptions { - dummy_identity: IDS["leader"], - dummy_auth: Default::default(), - dummy_auth_tokens: Default::default(), - }, - &config, - ); - - let redis = TESTS_POOL.get().await.expect("Should return Object"); - let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); - - setup_test_migrations(database.pool.clone()) - .await - .expect("Migrations should succeed"); - - let app = Application::new( - adapter, - config, - discard_logger(), - redis.connection.clone(), - database.pool.clone(), - ); - - app - } - #[tokio::test] async fn campaign_loading() { - let app = setup_app().await; + let app = setup_dummy_app().await; let build_request = |params: RouteParams| { Request::builder() From 88c324331be45e4ee463487f3d8e036f7882a3c6 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 4 Aug 2021 17:36:46 +0300 Subject: [PATCH 36/49] sentry - db - campaign - CampaignRemaining redis struct --- sentry/src/db/campaign.rs | 251 +++++++++++++++++++++++++++++++++- sentry/src/routes/campaign.rs | 233 ++++++++----------------------- 2 files changed, 307 insertions(+), 177 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index fed600754..9c5287a20 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -2,10 +2,16 @@ use crate::db::{DbPool, PoolError}; use primitives::{Campaign, CampaignId, ChannelId}; use tokio_postgres::types::Json; +pub use campaign_remaining::CampaignRemaining; + +/// ```text +/// INSERT INTO campaigns (id, channel_id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to) +/// VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) +/// ``` pub async fn insert_campaign(pool: &DbPool, campaign: &Campaign) -> Result { let client = pool.get().await?; let ad_units = Json(campaign.ad_units.clone()); - let stmt = client.prepare("INSERT INTO campaigns (id, channel_id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)").await?; + let stmt = client.prepare("INSERT INTO campaigns (id, channel_id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)").await?; let inserted = client .execute( &stmt, @@ -49,6 +55,10 @@ pub async fn fetch_campaign( } // TODO: We might need to use LIMIT to implement pagination +/// ```text +/// SELECT id, channel, creator, budget, validators, title, pricing_bounds, event_submission, ad_units, targeting_rules, created, active_from, active_to +/// FROM campaigns WHERE channel_id = $1 +/// ``` pub async fn get_campaigns_by_channel( pool: &DbPool, channel_id: &ChannelId, @@ -95,6 +105,245 @@ pub async fn update_campaign(pool: &DbPool, campaign: &Campaign) -> Result String { + format!("{}:{}", Self::CAMPAIGN_REMAINING_KEY, campaign) + } + + pub fn new(redis: MultiplexedConnection) -> Self { + Self { redis } + } + + pub async fn set_initial( + &self, + campaign: CampaignId, + amount: UnifiedNum, + ) -> Result { + redis::cmd("SETNX") + .arg(&Self::get_key(campaign)) + .arg(amount.to_u64()) + .query_async(&mut self.redis.clone()) + .await + } + + pub async fn get_remaining_opt( + &self, + campaign: CampaignId, + ) -> Result, RedisError> { + redis::cmd("GET") + .arg(&Self::get_key(campaign)) + .query_async::<_, Option>(&mut self.redis.clone()) + .await + } + + /// This method uses `max(0, value)` to clamp the value of a campaign, which can be negative and uses `i64`. + /// In addition, it defaults the campaign keys that were not found to `0`. + pub async fn get_multiple( + &self, + campaigns: &[CampaignId], + ) -> Result, RedisError> { + let keys: Vec = campaigns + .iter() + .map(|campaign| Self::get_key(*campaign)) + .collect(); + + let campaigns_remaining = redis::cmd("MGET") + .arg(keys) + .query_async::<_, Vec>>(&mut self.redis.clone()) + .await? + .into_iter() + .map(|remaining| match remaining { + Some(remaining) => UnifiedNum::from_u64(remaining.max(0).unsigned_abs()), + None => UnifiedNum::from_u64(0), + }) + .collect(); + + Ok(campaigns_remaining) + } + + pub async fn increase_by( + &self, + campaign: CampaignId, + amount: UnifiedNum, + ) -> Result { + let key = Self::get_key(campaign); + redis::cmd("INCRBY") + .arg(&key) + .arg(amount.to_u64()) + .query_async(&mut self.redis.clone()) + .await + } + + pub async fn decrease_by( + &self, + campaign: CampaignId, + amount: UnifiedNum, + ) -> Result { + let key = Self::get_key(campaign); + redis::cmd("DECRBY") + .arg(&key) + .arg(amount.to_u64()) + .query_async(&mut self.redis.clone()) + .await + } + } + + #[cfg(test)] + mod test { + use primitives::util::tests::prep_db::DUMMY_CAMPAIGN; + + use crate::db::redis_pool::TESTS_POOL; + + use super::*; + + #[tokio::test] + async fn it_sets_initial_increases_and_decreases_remaining_for_campaign() { + let redis = TESTS_POOL.get().await.expect("Should return Object"); + + let campaign = DUMMY_CAMPAIGN.id; + let campaign_remaining = CampaignRemaining::new(redis.connection.clone()); + + // Get remaining on a key which was not set + { + let get_remaining = campaign_remaining + .get_remaining_opt(campaign) + .await + .expect("Should get None"); + + assert_eq!(None, get_remaining); + } + + // Set Initial amount on that key + { + let initial_amount = UnifiedNum::from(1_000_u64); + let set_initial = campaign_remaining + .set_initial(campaign, initial_amount) + .await + .expect("Should set value in redis"); + assert!(set_initial); + + // get the remaining of that key, should be the initial value + let get_remaining = campaign_remaining + .get_remaining_opt(campaign) + .await + .expect("Should get None"); + + assert_eq!( + Some(1_000_i64), + get_remaining, + "should return the initial value that was set" + ); + } + + // Set initial on already existing key, should return `false` + { + let set_failing_initial = campaign_remaining + .set_initial(campaign, UnifiedNum::from(999_u64)) + .await + .expect("Should set value in redis"); + assert!(!set_failing_initial); + } + + // Decrease by amount + { + let decrease_amount = UnifiedNum::from(64); + let decrease_by = campaign_remaining + .decrease_by(campaign, decrease_amount) + .await + .expect("Should decrease remaining amount"); + + assert_eq!(936_i64, decrease_by); + } + + // Increase by amount + { + let increase_amount = UnifiedNum::from(1064); + let increase_by = campaign_remaining + .increase_by(campaign, increase_amount) + .await + .expect("Should increase remaining amount"); + + assert_eq!(2_000_i64, increase_by); + } + + let get_remaining = campaign_remaining + .get_remaining_opt(campaign) + .await + .expect("Should get remaining"); + + assert_eq!(Some(2_000_i64), get_remaining); + + // Decrease by amount > than currently set + { + let decrease_amount = UnifiedNum::from(5_000); + let decrease_by = campaign_remaining + .decrease_by(campaign, decrease_amount) + .await + .expect("Should decrease remaining amount"); + + assert_eq!(-3_000_i64, decrease_by); + } + + // Increase the negative value without going > 0 + { + let increase_amount = UnifiedNum::from(1000); + let increase_by = campaign_remaining + .increase_by(campaign, increase_amount) + .await + .expect("Should increase remaining amount"); + + assert_eq!(-2_000_i64, increase_by); + } + } + + #[tokio::test] + async fn it_gets_multiple_campaigns_remaining() { + let redis = TESTS_POOL.get().await.expect("Should return Object"); + let campaign_remaining = CampaignRemaining::new(redis.connection.clone()); + + let campaigns = (CampaignId::new(), CampaignId::new(), CampaignId::new()); + + // set initial amounts + { + assert!(campaign_remaining + .set_initial(campaigns.0, UnifiedNum::from(100)) + .await + .expect("Should set value in redis")); + + assert!(campaign_remaining + .set_initial(campaigns.1, UnifiedNum::from(200)) + .await + .expect("Should set value in redis")); + + assert!(campaign_remaining + .set_initial(campaigns.2, UnifiedNum::from(300)) + .await + .expect("Should set value in redis")); + } + + // set campaigns.1 to negative value, should return `0` because of `max(value, 0)` + assert_eq!(-300_i64, campaign_remaining.decrease_by(campaigns.1, UnifiedNum::from(500)).await.expect("Should decrease remaining")); + + let multiple = campaign_remaining.get_multiple(&[campaigns.0, campaigns.1, campaigns.2]).await.expect("Should get multiple"); + + assert_eq!(vec![UnifiedNum::from(100), UnifiedNum::from(0), UnifiedNum::from(300)], multiple); + } + } +} + #[cfg(test)] mod test { use primitives::{ diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index f7612dd04..6d57607b0 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,14 +1,4 @@ -use crate::{ - access::{self, check_access}, - db::{ - accounting::get_accounting_spent, - campaign::{get_campaigns_by_channel, insert_campaign, update_campaign}, - spendable::fetch_spendable, - DbPool, - }, - routes::campaign::update_campaign::set_initial_remaining_for_campaign, - success_response, Application, Auth, ResponseError, Session, -}; +use crate::{Application, Auth, ResponseError, Session, access::{self, check_access}, db::{CampaignRemaining, DbPool, RedisError, accounting::get_accounting_spent, campaign::{get_campaigns_by_channel, insert_campaign, update_campaign}, spendable::fetch_spendable}, success_response}; use chrono::Utc; use deadpool_postgres::PoolError; use hyper::{Body, Request, Response}; @@ -19,9 +9,8 @@ use primitives::{ campaign_create::{CreateCampaign, ModifyCampaign}, Event, SuccessResponse, }, - Address, Campaign, CampaignId, UnifiedNum, + Address, Campaign, UnifiedNum, }; -use redis::{aio::MultiplexedConnection, RedisError}; use slog::error; use std::{ cmp::{max, Ordering}, @@ -69,7 +58,7 @@ pub async fn create_campaign( campaign .validate(&app.config, &app.adapter.whoami()) - .map_err(|_| ResponseError::FailedValidation("couldn't valdiate campaign".to_string()))?; + .map_err(|err| ResponseError::FailedValidation(err.to_string()))?; if auth.uid.to_address() != campaign.creator { return Err(ResponseError::Forbidden( @@ -105,14 +94,22 @@ pub async fn create_campaign( } // If the campaign is being created, the amount spent is 0, therefore remaining = budget - set_initial_remaining_for_campaign(&app.redis, campaign.id, campaign.budget) + let remaining_set = CampaignRemaining::new(app.redis.clone()).set_initial(campaign.id, campaign.budget) .await .map_err(|_| { ResponseError::BadRequest( - "Couldn't update remaining while creating campaign".to_string(), + "Couldn't set remaining while creating campaign".to_string(), ) })?; + // If for some reason the randomly generated `CampaignId` exists in Redis + // This should **NOT** happen! + if !remaining_set { + return Err(ResponseError::Conflict( + "The generated CampaignId already exists, please repeat the request".to_string(), + )) + } + // insert Campaign match insert_campaign(&app.pool, &campaign).await { Err(error) => { @@ -136,50 +133,9 @@ pub async fn create_campaign( } pub mod update_campaign { - use super::*; - - pub const CAMPAIGN_REMAINING_KEY: &'static str = "campaignRemaining"; - - pub async fn set_initial_remaining_for_campaign( - redis: &MultiplexedConnection, - id: CampaignId, - amount: UnifiedNum, - ) -> Result { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); - redis::cmd("SETNX") - .arg(&key) - .arg(amount.to_u64()) - .query_async(&mut redis.clone()) - .await?; - Ok(true) - } - - pub async fn increase_remaining_for_campaign( - redis: &MultiplexedConnection, - id: CampaignId, - amount: UnifiedNum, - ) -> Result { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); - redis::cmd("INCRBY") - .arg(&key) - .arg(amount.to_u64()) - .query_async::<_, u64>(&mut redis.clone()) - .await - .map(UnifiedNum::from) - } + use crate::db::CampaignRemaining; - pub async fn decrease_remaining_for_campaign( - redis: &MultiplexedConnection, - id: CampaignId, - amount: UnifiedNum, - ) -> Result { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); - redis::cmd("DECRBY") - .arg(&key) - .arg(amount.to_u64()) - .query_async::<_, i64>(&mut redis.clone()) - .await - } + use super::*; pub async fn handle_route( req: Request, @@ -196,10 +152,12 @@ pub mod update_campaign { let modify_campaign_fields = serde_json::from_slice::(&body) .map_err(|e| ResponseError::FailedValidation(e.to_string()))?; + let campaign_remaining = CampaignRemaining::new(app.redis.clone()); + // modify Campaign let modified_campaign = modify_campaign( &app.pool, - &mut app.redis.clone(), + campaign_remaining, campaign_being_mutated, modify_campaign_fields, ) @@ -211,7 +169,7 @@ pub mod update_campaign { pub async fn modify_campaign( pool: &DbPool, - redis: &MultiplexedConnection, + campaign_remaining: CampaignRemaining, campaign: Campaign, modify_campaign: ModifyCampaign, ) -> Result { @@ -220,7 +178,7 @@ pub mod update_campaign { // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 let delta_budget = if let Some(new_budget) = modify_campaign.budget { - get_delta_budget(redis, &campaign, new_budget).await? + get_delta_budget(&campaign_remaining, &campaign, new_budget).await? } else { None }; @@ -245,15 +203,19 @@ pub mod update_campaign { let total_remaining = total_deposited .checked_sub(&accounting_spent) .ok_or(Error::Calculation)?; - let channel_campaigns = get_campaigns_by_channel(&pool, &campaign.channel.id()).await?; + let channel_campaigns = get_campaigns_by_channel(&pool, &campaign.channel.id()) + .await? + .iter() + .map(|c| c.id) + .collect::>(); // this will include the Campaign we are currently modifying - let campaigns_current_remaining_sum = - get_remaining_for_multiple_campaigns(&redis, &channel_campaigns) - .await? - .iter() - .sum::>() - .ok_or(Error::Calculation)?; + let campaigns_current_remaining_sum = campaign_remaining + .get_multiple(&channel_campaigns) + .await? + .iter() + .sum::>() + .ok_or(Error::Calculation)?; // apply the delta_budget to the sum let new_campaigns_remaining = match delta_budget { @@ -270,19 +232,18 @@ pub mod update_campaign { return Err(Error::CampaignNotModified); } - // if the value is not positive it will return an error because of UnifiedNum + // there is a chance that the new remaining will be negative even when increasing the budget + // We don't currently use this value but can be used to perform additional checks or return messages accordingly let _campaign_remaining = match delta_budget { - // should always be positive DeltaBudget::Increase(increase_by) => { - increase_remaining_for_campaign(redis, campaign.id, increase_by).await? + campaign_remaining + .increase_by(campaign.id, increase_by) + .await? } - // there is a chance that an even lowered the remaining and it's no longer positive - // check if positive and create an UnifiedNum, or return an error DeltaBudget::Decrease(decrease_by) => { - match decrease_remaining_for_campaign(redis, campaign.id, decrease_by).await? { - remaining if remaining >= 0 => UnifiedNum::from(remaining.unsigned_abs()), - _ => UnifiedNum::from(0), - } + campaign_remaining + .decrease_by(campaign.id, decrease_by) + .await? } }; } @@ -303,7 +264,7 @@ pub mod update_campaign { } async fn get_delta_budget( - redis: &MultiplexedConnection, + campaign_remaining: &CampaignRemaining, campaign: &Campaign, new_budget: UnifiedNum, ) -> Result>, Error> { @@ -316,8 +277,10 @@ pub mod update_campaign { Ordering::Less => DeltaBudget::Decrease(()), }; - let old_remaining = get_remaining_for_campaign(redis, campaign.id) + let old_remaining = campaign_remaining + .get_remaining_opt(campaign.id) .await? + .map(|remaining| UnifiedNum::from(max(0, remaining).unsigned_abs())) .ok_or(Error::FailedUpdate( "No remaining entry for campaign".to_string(), ))?; @@ -362,44 +325,6 @@ pub mod update_campaign { Ok(Some(budget)) } - - pub async fn get_remaining_for_campaign( - redis: &MultiplexedConnection, - id: CampaignId, - ) -> Result, RedisError> { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, id); - - let remaining = redis::cmd("GET") - .arg(&key) - .query_async::<_, Option>(&mut redis.clone()) - .await? - .map(|remaining| UnifiedNum::from(max(0, remaining).unsigned_abs())); - - Ok(remaining) - } - - async fn get_remaining_for_multiple_campaigns( - redis: &MultiplexedConnection, - campaigns: &[Campaign], - ) -> Result, Error> { - let keys: Vec = campaigns - .iter() - .map(|c| format!("{}:{}", CAMPAIGN_REMAINING_KEY, c.id)) - .collect(); - - let remainings = redis::cmd("MGET") - .arg(keys) - .query_async::<_, Vec>>(&mut redis.clone()) - .await? - .into_iter() - .map(|remaining| match remaining { - Some(remaining) => UnifiedNum::from_u64(max(0, remaining).unsigned_abs()), - None => UnifiedNum::from_u64(0), - }) - .collect(); - - Ok(remainings) - } } pub async fn insert_events( @@ -479,67 +404,23 @@ async fn process_events( mod test { use super::*; use crate::{ - campaign::update_campaign::{increase_remaining_for_campaign, CAMPAIGN_REMAINING_KEY}, - db::redis_pool::TESTS_POOL, + db::{redis_pool::TESTS_POOL, tests_postgres::DATABASE_POOL}, }; + use adapter::DummyAdapter; use primitives::util::tests::prep_db::DUMMY_CAMPAIGN; #[tokio::test] - async fn does_it_increase_remaining() { - let mut redis = TESTS_POOL.get().await.expect("Should return Object"); - let campaign = DUMMY_CAMPAIGN.clone(); - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign.id); - - // Setting the redis base variable - redis::cmd("SET") - .arg(&key) - .arg(100_u64) - .query_async::<_, ()>(&mut redis.connection) - .await - .expect("should set"); - - // 2 async calls at once, should be 500 after them - futures::future::try_join_all([ - increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)), - increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(200)), - ]) - .await - .expect("Should increase remaining twice"); - - let remaining = redis::cmd("GET") - .arg(&key) - .query_async::<_, Option>(&mut redis.connection) - .await - .expect("should get remaining"); - assert_eq!( - remaining.map(UnifiedNum::from_u64), - Some(UnifiedNum::from_u64(500)) - ); - - increase_remaining_for_campaign(&redis, campaign.id, campaign.budget) - .await - .expect("should increase"); - - let remaining = redis::cmd("GET") - .arg(&key) - // Directly parsing to u64 as we know it will be >0 - .query_async::<_, Option>(&mut redis.connection) - .await - .expect("should get remaining"); - - let should_be_remaining = UnifiedNum::from_u64(500) + campaign.budget; - assert_eq!(remaining.map(UnifiedNum::from), Some(should_be_remaining)); - - increase_remaining_for_campaign(&redis, campaign.id, UnifiedNum::from_u64(0)) - .await - .expect("should increase remaining"); - - let remaining = redis::cmd("GET") - .arg(&key) - .query_async::<_, Option>(&mut redis.connection) - .await - .expect("should get remaining"); - - assert_eq!(remaining.map(UnifiedNum::from), Some(should_be_remaining)); + /// Test single campaign creation and modification + // & + /// Test with multiple campaigns (because of Budget) a modification of campaign + async fn create_and_modify_with_multiple_campaigns() { + todo!() + // let redis = TESTS_POOL.get().await.expect("Should get redis"); + // let postgres = DATABASE_POOL.get().await.expect("Should get postgres"); + // let campaign = DUMMY_CAMPAIGN.clone(); + + // let app = Application::new() + + } } From 3ca6ab56264fb5ca6c23b149706ebe6adc652ed6 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 4 Aug 2021 18:41:40 +0300 Subject: [PATCH 37/49] primitives - sentry - impl From for CreateCampaign --- primitives/src/sentry.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index 6018dc235..ba07b4035 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -361,6 +361,26 @@ pub mod campaign_create { } } + /// This implementation helps with test setup + /// **NOTE:** It erases the CampaignId, since the creation of the campaign gives it's CampaignId + impl From for CreateCampaign { + fn from(campaign: Campaign) -> Self { + Self { + channel: campaign.channel, + creator: campaign.creator, + budget: campaign.budget, + validators: campaign.validators, + title: campaign.title, + pricing_bounds: campaign.pricing_bounds, + event_submission: campaign.event_submission, + ad_units: campaign.ad_units, + targeting_rules: campaign.targeting_rules, + created: campaign.created, + active: campaign.active, + } + } + } + // All editable fields stored in one place, used for checking when a budget is changed #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ModifyCampaign { @@ -403,7 +423,7 @@ pub mod campaign_create { if let Some(new_pricing_bounds) = self.pricing_bounds { campaign.pricing_bounds = Some(new_pricing_bounds); } - + if let Some(new_event_submission) = self.event_submission { campaign.event_submission = Some(new_event_submission); } From 11dc71564d960796801c967aa92014d7ba0cbff2 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Wed, 4 Aug 2021 18:42:05 +0300 Subject: [PATCH 38/49] primitives - validator - From
for ValidatorID --- primitives/src/validator.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/primitives/src/validator.rs b/primitives/src/validator.rs index 81452b5c9..72b89f7a1 100644 --- a/primitives/src/validator.rs +++ b/primitives/src/validator.rs @@ -39,6 +39,12 @@ impl From<&Address> for ValidatorId { } } +impl From
for ValidatorId { + fn from(address: Address) -> Self { + Self(address) + } +} + impl From<&[u8; 20]> for ValidatorId { fn from(bytes: &[u8; 20]) -> Self { Self(Address::from(bytes)) From 07d9c74c969b243060800d0f883800e3799bb46d Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 11:12:25 +0300 Subject: [PATCH 39/49] primitives - sentry - accounting - Balances - add spender/earner --- primitives/src/sentry/accounting.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/primitives/src/sentry/accounting.rs b/primitives/src/sentry/accounting.rs index 4b9256cec..e04167eb9 100644 --- a/primitives/src/sentry/accounting.rs +++ b/primitives/src/sentry/accounting.rs @@ -68,6 +68,16 @@ impl Balances { Ok(()) } + + /// Adds the spender to the Balances with `UnifiedNum::from(0)` if he does not exist + pub fn add_spender(&mut self, spender: Address) { + self.spenders.entry(spender).or_insert(UnifiedNum::from(0)); + } + + /// Adds the earner to the Balances with `UnifiedNum::from(0)` if he does not exist + pub fn add_earner(&mut self, earner: Address) { + self.earners.entry(earner).or_insert(UnifiedNum::from(0)); + } } #[derive(Debug)] From d80e109c61fbefe392f091f32cd883b5bca81ea6 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 11:13:31 +0300 Subject: [PATCH 40/49] sentry - Applicaiton - add CampaignRemaining to app --- sentry/src/lib.rs | 17 +++++++++++------ sentry/src/main.rs | 7 ++++--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index ca5bde526..0d2126a76 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -7,6 +7,7 @@ use crate::routes::channel::channel_status; use crate::routes::event_aggregate::list_channel_event_aggregates; use crate::routes::validator_message::{extract_params, list_validator_messages}; use chrono::Utc; +use db::CampaignRemaining; use hyper::{Body, Method, Request, Response, StatusCode}; use lazy_static::lazy_static; use middleware::{ @@ -90,10 +91,11 @@ impl RouteParams { #[derive(Clone)] pub struct Application { pub adapter: A, + pub config: Config, pub logger: Logger, pub redis: MultiplexedConnection, pub pool: DbPool, - pub config: Config, + pub campaign_remaining: CampaignRemaining, } impl Application { @@ -103,6 +105,7 @@ impl Application { logger: Logger, redis: MultiplexedConnection, pool: DbPool, + campaign_remaining: CampaignRemaining ) -> Self { Self { adapter, @@ -110,6 +113,7 @@ impl Application { logger, redis, pool, + campaign_remaining, } } @@ -513,13 +517,11 @@ pub mod test_util { }, }; - use crate::{Application, db::{ - redis_pool::TESTS_POOL, - tests_postgres::{setup_test_migrations, DATABASE_POOL}, - }}; + use crate::{Application, db::{CampaignRemaining, redis_pool::TESTS_POOL, tests_postgres::{setup_test_migrations, DATABASE_POOL}}}; + /// Uses production configuration to setup the correct Contract addresses for tokens. pub async fn setup_dummy_app() -> Application { - let config = configuration("development", None).expect("Should get Config"); + let config = configuration("production", None).expect("Should get Config"); let adapter = DummyAdapter::init( DummyAdapterOptions { dummy_identity: IDS["leader"], @@ -536,12 +538,15 @@ pub mod test_util { .await .expect("Migrations should succeed"); + let campaign_remaining = CampaignRemaining::new(redis.connection.clone()); + let app = Application::new( adapter, config, discard_logger(), redis.connection.clone(), database.pool.clone(), + campaign_remaining ); app diff --git a/sentry/src/main.rs b/sentry/src/main.rs index f8963f562..634831e5d 100644 --- a/sentry/src/main.rs +++ b/sentry/src/main.rs @@ -10,7 +10,7 @@ use primitives::adapter::{Adapter, DummyAdapterOptions, KeystoreOptions}; use primitives::config::configuration; use primitives::util::tests::prep_db::{AUTH, IDS}; use primitives::ValidatorId; -use sentry::db::{postgres_connection, redis_connection, setup_migrations}; +use sentry::db::{CampaignRemaining, postgres_connection, redis_connection, setup_migrations}; use sentry::Application; use slog::{error, info, Logger}; use std::{ @@ -115,18 +115,19 @@ async fn main() -> Result<(), Box> { // Check connection and setup migrations before setting up Postgres setup_migrations(&environment).await; let postgres = postgres_connection(42).await; + let campaign_remaining = CampaignRemaining::new(redis.clone()); match adapter { AdapterTypes::EthereumAdapter(adapter) => { run( - Application::new(*adapter, config, logger, redis, postgres), + Application::new(*adapter, config, logger, redis, postgres, campaign_remaining), socket_addr, ) .await } AdapterTypes::DummyAdapter(adapter) => { run( - Application::new(*adapter, config, logger, redis, postgres), + Application::new(*adapter, config, logger, redis, postgres, campaign_remaining), socket_addr, ) .await From e1a245ed20d7008f471a4cdca7462587b001ad39 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 11:14:41 +0300 Subject: [PATCH 41/49] sentry - db - CampaignRemaining - MGET guard against empty vec of campaigns --- sentry/src/db/campaign.rs | 54 +++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/sentry/src/db/campaign.rs b/sentry/src/db/campaign.rs index 9c5287a20..ea24ec6f7 100644 --- a/sentry/src/db/campaign.rs +++ b/sentry/src/db/campaign.rs @@ -155,6 +155,11 @@ mod campaign_remaining { &self, campaigns: &[CampaignId], ) -> Result, RedisError> { + // `MGET` fails on empty keys + if campaigns.is_empty() { + return Ok(vec![]); + } + let keys: Vec = campaigns .iter() .map(|campaign| Self::get_key(*campaign)) @@ -314,6 +319,17 @@ mod campaign_remaining { let redis = TESTS_POOL.get().await.expect("Should return Object"); let campaign_remaining = CampaignRemaining::new(redis.connection.clone()); + // get multiple with empty campaigns slice + // `MGET` throws error on an empty keys argument + assert!( + campaign_remaining + .get_multiple(&[]) + .await + .expect("Should get multiple") + .is_empty(), + "Should return an empty result" + ); + let campaigns = (CampaignId::new(), CampaignId::new(), CampaignId::new()); // set initial amounts @@ -323,23 +339,39 @@ mod campaign_remaining { .await .expect("Should set value in redis")); - assert!(campaign_remaining - .set_initial(campaigns.1, UnifiedNum::from(200)) - .await - .expect("Should set value in redis")); + assert!(campaign_remaining + .set_initial(campaigns.1, UnifiedNum::from(200)) + .await + .expect("Should set value in redis")); - assert!(campaign_remaining - .set_initial(campaigns.2, UnifiedNum::from(300)) - .await - .expect("Should set value in redis")); + assert!(campaign_remaining + .set_initial(campaigns.2, UnifiedNum::from(300)) + .await + .expect("Should set value in redis")); } // set campaigns.1 to negative value, should return `0` because of `max(value, 0)` - assert_eq!(-300_i64, campaign_remaining.decrease_by(campaigns.1, UnifiedNum::from(500)).await.expect("Should decrease remaining")); + assert_eq!( + -300_i64, + campaign_remaining + .decrease_by(campaigns.1, UnifiedNum::from(500)) + .await + .expect("Should decrease remaining") + ); - let multiple = campaign_remaining.get_multiple(&[campaigns.0, campaigns.1, campaigns.2]).await.expect("Should get multiple"); + let multiple = campaign_remaining + .get_multiple(&[campaigns.0, campaigns.1, campaigns.2]) + .await + .expect("Should get multiple"); - assert_eq!(vec![UnifiedNum::from(100), UnifiedNum::from(0), UnifiedNum::from(300)], multiple); + assert_eq!( + vec![ + UnifiedNum::from(100), + UnifiedNum::from(0), + UnifiedNum::from(300) + ], + multiple + ); } } } From 9a94c5f1fad1e8577c027c7c2ba7528b381c3db1 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 11:15:27 +0300 Subject: [PATCH 42/49] sentry - routes - campaign create/modify - more checks & tests --- sentry/src/routes/campaign.rs | 310 +++++++++++++++++++++++++++++----- 1 file changed, 269 insertions(+), 41 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 6d57607b0..7fd2b9b50 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,4 +1,13 @@ -use crate::{Application, Auth, ResponseError, Session, access::{self, check_access}, db::{CampaignRemaining, DbPool, RedisError, accounting::get_accounting_spent, campaign::{get_campaigns_by_channel, insert_campaign, update_campaign}, spendable::fetch_spendable}, success_response}; +use crate::{ + access::{self, check_access}, + db::{ + accounting::get_accounting_spent, + campaign::{get_campaigns_by_channel, insert_campaign, update_campaign}, + spendable::fetch_spendable, + CampaignRemaining, DbPool, RedisError, + }, + success_response, Application, Auth, ResponseError, Session, +}; use chrono::Utc; use deadpool_postgres::PoolError; use hyper::{Body, Request, Response}; @@ -69,37 +78,55 @@ pub async fn create_campaign( let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); - let accounting_spent = - get_accounting_spent(app.pool.clone(), &campaign.creator, &campaign.channel.id()).await?; + let total_remaining = + { + let accounting_spent = + get_accounting_spent(app.pool.clone(), &campaign.creator, &campaign.channel.id()) + .await?; - let latest_spendable = - fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()) - .await? - .ok_or(ResponseError::BadRequest( - "No spendable amount found for the Campaign creator".to_string(), - ))?; - let total_deposited = latest_spendable.deposit.total; + let latest_spendable = + fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()) + .await? + .ok_or(ResponseError::BadRequest( + "No spendable amount found for the Campaign creator".to_string(), + ))?; + // Gets the latest Spendable for this (spender, channelId) pair + let total_deposited = latest_spendable.deposit.total; - let remaining_for_channel = - total_deposited - .checked_sub(&accounting_spent) - .ok_or(ResponseError::FailedValidation( - "No more budget remaining".to_string(), - ))?; + total_deposited.checked_sub(&accounting_spent).ok_or( + ResponseError::FailedValidation("No more budget remaining".to_string()), + )? + }; - if campaign.budget > remaining_for_channel { + let channel_campaigns = get_campaigns_by_channel(&app.pool, &campaign.channel.id()) + .await? + .iter() + .map(|c| c.id) + .collect::>(); + + let campaigns_remaining_sum = app + .campaign_remaining + .get_multiple(&channel_campaigns) + .await? + .iter() + .sum::>() + .ok_or(Error::Calculation)? + // DO NOT FORGET to add the Campaign being created right now! + .checked_add(&campaign.budget) + .ok_or(Error::Calculation)?; + + if !(campaigns_remaining_sum <= total_remaining) || campaign.budget > total_remaining { return Err(ResponseError::BadRequest( - "Not enough deposit left for the new campaign budget".to_string(), + "Not enough deposit left for the new campaign's budget".to_string(), )); } // If the campaign is being created, the amount spent is 0, therefore remaining = budget - let remaining_set = CampaignRemaining::new(app.redis.clone()).set_initial(campaign.id, campaign.budget) + let remaining_set = CampaignRemaining::new(app.redis.clone()) + .set_initial(campaign.id, campaign.budget) .await .map_err(|_| { - ResponseError::BadRequest( - "Couldn't set remaining while creating campaign".to_string(), - ) + ResponseError::BadRequest("Couldn't set remaining while creating campaign".to_string()) })?; // If for some reason the randomly generated `CampaignId` exists in Redis @@ -107,7 +134,7 @@ pub async fn create_campaign( if !remaining_set { return Err(ResponseError::Conflict( "The generated CampaignId already exists, please repeat the request".to_string(), - )) + )); } // insert Campaign @@ -152,12 +179,10 @@ pub mod update_campaign { let modify_campaign_fields = serde_json::from_slice::(&body) .map_err(|e| ResponseError::FailedValidation(e.to_string()))?; - let campaign_remaining = CampaignRemaining::new(app.redis.clone()); - // modify Campaign let modified_campaign = modify_campaign( &app.pool, - campaign_remaining, + &app.campaign_remaining, campaign_being_mutated, modify_campaign_fields, ) @@ -169,7 +194,7 @@ pub mod update_campaign { pub async fn modify_campaign( pool: &DbPool, - campaign_remaining: CampaignRemaining, + campaign_remaining: &CampaignRemaining, campaign: Campaign, modify_campaign: ModifyCampaign, ) -> Result { @@ -178,7 +203,7 @@ pub mod update_campaign { // *NOTE*: To close a campaign set campaignBudget to campaignSpent so that spendable == 0 let delta_budget = if let Some(new_budget) = modify_campaign.budget { - get_delta_budget(&campaign_remaining, &campaign, new_budget).await? + get_delta_budget(campaign_remaining, &campaign, new_budget).await? } else { None }; @@ -229,7 +254,9 @@ pub mod update_campaign { .ok_or(Error::Calculation)?; if !(new_campaigns_remaining <= total_remaining) { - return Err(Error::CampaignNotModified); + return Err(Error::NewBudget( + "Not enough deposit left for the campaign's new budget".to_string(), + )); } // there is a chance that the new remaining will be negative even when increasing the budget @@ -402,25 +429,226 @@ async fn process_events( #[cfg(test)] mod test { - use super::*; + use super::{update_campaign::modify_campaign, *}; use crate::{ - db::{redis_pool::TESTS_POOL, tests_postgres::DATABASE_POOL}, + db::{accounting::insert_accounting, spendable::insert_spendable}, + test_util::setup_dummy_app, + }; + use hyper::StatusCode; + use primitives::{ + sentry::accounting::{Balances, CheckedState}, + spender::{Deposit, Spendable}, + util::tests::prep_db::DUMMY_CAMPAIGN, + ValidatorId, }; - use adapter::DummyAdapter; - use primitives::util::tests::prep_db::DUMMY_CAMPAIGN; #[tokio::test] /// Test single campaign creation and modification // & /// Test with multiple campaigns (because of Budget) a modification of campaign async fn create_and_modify_with_multiple_campaigns() { - todo!() - // let redis = TESTS_POOL.get().await.expect("Should get redis"); - // let postgres = DATABASE_POOL.get().await.expect("Should get postgres"); - // let campaign = DUMMY_CAMPAIGN.clone(); - - // let app = Application::new() - - + let app = setup_dummy_app().await; + + let build_request = |create_campaign: CreateCampaign| -> Request { + let auth = Auth { + era: 0, + uid: ValidatorId::from(create_campaign.creator), + }; + + let body = + Body::from(serde_json::to_string(&create_campaign).expect("Should serialize")); + + Request::builder() + .extension(auth) + .body(body) + .expect("Should build Request") + }; + + let campaign: Campaign = { + // erases the CampaignId for the CreateCampaign request + let mut create = CreateCampaign::from(DUMMY_CAMPAIGN.clone()); + // 500.00000000 + create.budget = UnifiedNum::from(50_000_000_000); + + let spendable = Spendable { + spender: create.creator, + channel: create.channel.clone(), + deposit: Deposit { + // a deposit equal to double the Campaign Budget + total: UnifiedNum::from(200_000_000_000), + still_on_create2: UnifiedNum::from(0), + }, + }; + assert!(insert_spendable(app.pool.clone(), &spendable) + .await + .expect("Should insert Spendable for Campaign creator")); + + let mut balances = Balances::::default(); + balances.add_spender(create.creator); + + // TODO: Replace this once https://github.com/AdExNetwork/adex-validator-stack-rust/pull/413 is merged + let _accounting = insert_accounting(app.pool.clone(), create.channel.clone(), balances) + .await + .expect("Should create Accounting"); + + let create_response = create_campaign(build_request(create), &app) + .await + .expect("Should create campaign"); + + assert_eq!(StatusCode::OK, create_response.status()); + let json = hyper::body::to_bytes(create_response.into_body()) + .await + .expect("Should get json"); + + let campaign: Campaign = + serde_json::from_slice(&json).expect("Should get new Campaign"); + + assert_ne!(DUMMY_CAMPAIGN.id, campaign.id); + + let campaign_remaining = CampaignRemaining::new(app.redis.clone()); + + let remaining = campaign_remaining + .get_remaining_opt(campaign.id) + .await + .expect("Should get remaining from redis") + .expect("There should be value for the Campaign"); + + assert_eq!( + UnifiedNum::from(50_000_000_000), + UnifiedNum::from(remaining.unsigned_abs()) + ); + campaign + }; + + // modify campaign + let modified = { + // 1000.00000000 + let new_budget = UnifiedNum::from(100_000_000_000); + let modify = ModifyCampaign { + budget: Some(new_budget.clone()), + validators: None, + title: Some("Updated title".to_string()), + pricing_bounds: None, + event_submission: None, + ad_units: None, + targeting_rules: None, + }; + + let modified_campaign = + modify_campaign(&app.pool, &app.campaign_remaining, campaign.clone(), modify) + .await + .expect("Should modify campaign"); + + assert_eq!(new_budget, modified_campaign.budget); + assert_eq!(Some("Updated title".to_string()), modified_campaign.title); + + modified_campaign + }; + + // we have 1000 left from our deposit, so we are using half of it + let _second_campaign = { + // erases the CampaignId for the CreateCampaign request + let mut create_second = CreateCampaign::from(DUMMY_CAMPAIGN.clone()); + // 500.00000000 + create_second.budget = UnifiedNum::from(50_000_000_000); + + let create_response = create_campaign(build_request(create_second), &app) + .await + .expect("Should create campaign"); + + assert_eq!(StatusCode::OK, create_response.status()); + let json = hyper::body::to_bytes(create_response.into_body()) + .await + .expect("Should get json"); + + let second_campaign: Campaign = + serde_json::from_slice(&json).expect("Should get new Campaign"); + + second_campaign + }; + + // No budget left for new campaigns + // remaining: 500 + // new campaign budget: 600 + { + // erases the CampaignId for the CreateCampaign request + let mut create = CreateCampaign::from(DUMMY_CAMPAIGN.clone()); + // 600.00000000 + create.budget = UnifiedNum::from(60_000_000_000); + + let create_err = create_campaign(build_request(create), &app) + .await + .expect_err("Should return Error response"); + + assert_eq!(ResponseError::BadRequest("Not enough deposit left for the new campaign's budget".to_string()), create_err); + } + + // modify first campaign, by lowering the budget from 1000 to 900 + let modified = { + let lower_budget = UnifiedNum::from(90_000_000_000); + let modify = ModifyCampaign { + budget: Some(lower_budget.clone()), + validators: None, + title: None, + pricing_bounds: None, + event_submission: None, + ad_units: None, + targeting_rules: None, + }; + + let modified_campaign = + modify_campaign(&app.pool, &app.campaign_remaining, modified, modify) + .await + .expect("Should modify campaign"); + + assert_eq!(lower_budget, modified_campaign.budget); + + modified_campaign + }; + + // Just enough budget to create this Campaign + // remaining: 600 + // new campaign budget: 600 + { + // erases the CampaignId for the CreateCampaign request + let mut create = CreateCampaign::from(DUMMY_CAMPAIGN.clone()); + // 600.00000000 + create.budget = UnifiedNum::from(60_000_000_000); + + let create_response = create_campaign(build_request(create), &app) + .await + .expect("Should return create campaign"); + + let json = hyper::body::to_bytes(create_response.into_body()) + .await + .expect("Should get json"); + + let _campaign: Campaign = + serde_json::from_slice(&json).expect("Should get new Campaign"); + } + + // Modify a campaign without enough budget + // remaining: 0 + // new campaign budget: 1100 + // current campaign budget: 900 + { + let new_budget = UnifiedNum::from(110_000_000_000); + let modify = ModifyCampaign { + budget: Some(new_budget), + validators: None, + title: None, + pricing_bounds: None, + event_submission: None, + ad_units: None, + targeting_rules: None, + }; + + let modify_err = + modify_campaign(&app.pool, &app.campaign_remaining, modified, modify) + .await + .expect_err("Should return Error response"); + + assert!(matches!(modify_err, Error::NewBudget(string) if string == "Not enough deposit left for the campaign's new budget")); + } } } From 2752dea1b263a18e5bc98fbc1b74581662077387 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 12:02:26 +0300 Subject: [PATCH 43/49] fix PR comments --- sentry/src/db/accounting.rs | 13 +++++++------ sentry/src/routes/campaign.rs | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sentry/src/db/accounting.rs b/sentry/src/db/accounting.rs index ae79ac7bd..f09741a4b 100644 --- a/sentry/src/db/accounting.rs +++ b/sentry/src/db/accounting.rs @@ -11,6 +11,8 @@ use tokio_postgres::{ use super::{DbPool, PoolError}; use thiserror::Error; +static UPDATE_ACCOUNTING_STATEMENT: &str = "INSERT INTO accounting(channel_id, side, address, amount, updated, created) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT ON CONSTRAINT accounting_pkey DO UPDATE SET amount = accounting.amount + $4, updated = $6 WHERE accounting.channel_id = $1 AND accounting.side = $2 AND accounting.address = $3 RETURNING channel_id, side, address, amount, updated, created"; + #[derive(Debug, Error)] pub enum Error { #[error("Accounting Balances error: {0}")] @@ -84,7 +86,6 @@ pub async fn get_accounting( /// Will update current Spender/Earner amount or insert a new Accounting record /// /// See `UPDATE_ACCOUNTING_STATEMENT` static for full query. -static UPDATE_ACCOUNTING_STATEMENT: &str = "INSERT INTO accounting(channel_id, side, address, amount, updated, created) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT ON CONSTRAINT accounting_pkey DO UPDATE SET amount = accounting.amount + $4, updated = $6 WHERE accounting.channel_id = $1 AND accounting.side = $2 AND accounting.address = $3 RETURNING channel_id, side, address, amount, updated, created"; pub async fn update_accounting( pool: DbPool, channel_id: ChannelId, @@ -204,7 +205,7 @@ mod test { .expect("Should insert"); assert_eq!(spender, inserted.address); assert_eq!(Side::Spender, inserted.side); - assert_eq!(UnifiedNum::from(100_000_000), inserted.amount); + assert_eq!(amount, inserted.amount); let updated = update_accounting( database.pool.clone(), @@ -218,7 +219,7 @@ mod test { assert_eq!(spender, updated.address); assert_eq!(Side::Spender, updated.side); assert_eq!( - UnifiedNum::from(300_000_000), + amount + update_amount, updated.amount, "Should add the newly spent amount to the existing one" ); @@ -247,7 +248,7 @@ mod test { .expect("Should insert"); assert_eq!(earner, inserted.address); assert_eq!(Side::Earner, inserted.side); - assert_eq!(UnifiedNum::from(100_000_000), inserted.amount); + assert_eq!(amount, inserted.amount); let updated = update_accounting( database.pool.clone(), @@ -261,7 +262,7 @@ mod test { assert_eq!(earner, updated.address); assert_eq!(Side::Earner, updated.side); assert_eq!( - UnifiedNum::from(300_000_000), + amount + update_amount, updated.amount, "Should add the newly earned amount to the existing one" ); @@ -293,7 +294,7 @@ mod test { .expect("Should insert"); assert_eq!(spender_as_earner, inserted.address); assert_eq!(Side::Earner, inserted.side); - assert_eq!(UnifiedNum::from(100_000_000), inserted.amount); + assert_eq!(amount, inserted.amount); let updated = update_accounting( database.pool.clone(), diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index e9574f62f..e036a4db8 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -435,7 +435,7 @@ pub mod insert_events { .await; assert!( - dbg!(spend_event).is_ok(), + spend_event.is_ok(), "Campaign budget has no remaining funds to spend" ); From 5177217cd7d24ddf261a855983259dabf52311f5 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 12:46:50 +0300 Subject: [PATCH 44/49] sentry - db - make pub insert_accounting --- sentry/src/db/accounting.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sentry/src/db/accounting.rs b/sentry/src/db/accounting.rs index ffc8e3db4..94fcae16e 100644 --- a/sentry/src/db/accounting.rs +++ b/sentry/src/db/accounting.rs @@ -38,9 +38,7 @@ pub async fn get_accounting_spent( Ok(row.get("spent")) } -// TODO This is still WIP -#[allow(dead_code)] -async fn insert_accounting( +pub async fn insert_accounting( pool: DbPool, channel: Channel, balances: Balances, From 75a5a5f63350f4a1991fa7c209329f1b8b142f1c Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 15:46:53 +0300 Subject: [PATCH 45/49] sentry - routes - campaign - get_delta_budget - fix logic & naming --- sentry/src/routes/campaign.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 7fd2b9b50..b3db3abe1 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -330,6 +330,7 @@ pub mod update_campaign { .checked_sub(¤t_budget) .and_then(|delta_budget| old_remaining.checked_add(&delta_budget)) .ok_or(Error::Calculation)?; + // new remaining > old remaining let increase_by = new_remaining .checked_sub(&old_remaining) .ok_or(Error::Calculation)?; @@ -337,13 +338,14 @@ pub mod update_campaign { DeltaBudget::Increase(increase_by) } DeltaBudget::Decrease(()) => { - // delta budget = New budget - Old budget ( the difference between the new and old when New > Old) + // delta budget = Old budget - New budget ( the difference between the new and old when New < Old) let new_remaining = ¤t_budget .checked_sub(&new_budget) - .and_then(|delta_budget| old_remaining.checked_add(&delta_budget)) + .and_then(|delta_budget| old_remaining.checked_sub(&delta_budget)) .ok_or(Error::Calculation)?; - let decrease_by = new_remaining - .checked_sub(&old_remaining) + // old remaining > new remaining + let decrease_by = old_remaining + .checked_sub(&new_remaining) .ok_or(Error::Calculation)?; DeltaBudget::Decrease(decrease_by) From 70376b30313db79643bc1c69aacefb0b112b1212 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 9 Aug 2021 17:42:35 +0300 Subject: [PATCH 46/49] Fix merge issues & run rustfmt --- sentry/src/lib.rs | 18 ++++--- sentry/src/main.rs | 20 ++++++-- sentry/src/payout.rs | 4 +- sentry/src/routes/campaign.rs | 89 ++++++++++++++++++----------------- 4 files changed, 75 insertions(+), 56 deletions(-) diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 27c4be023..713aa80e7 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -105,7 +105,7 @@ impl Application { logger: Logger, redis: MultiplexedConnection, pool: DbPool, - campaign_remaining: CampaignRemaining + campaign_remaining: CampaignRemaining, ) -> Self { Self { adapter, @@ -511,14 +511,18 @@ pub mod test_util { use primitives::{ adapter::DummyAdapterOptions, config::configuration, - util::tests::{ - discard_logger, - prep_db::{IDS}, + util::tests::{discard_logger, prep_db::IDS}, + }; + + use crate::{ + db::{ + redis_pool::TESTS_POOL, + tests_postgres::{setup_test_migrations, DATABASE_POOL}, + CampaignRemaining, }, + Application, }; - use crate::{Application, db::{CampaignRemaining, redis_pool::TESTS_POOL, tests_postgres::{setup_test_migrations, DATABASE_POOL}}}; - /// Uses production configuration to setup the correct Contract addresses for tokens. pub async fn setup_dummy_app() -> Application { let config = configuration("production", None).expect("Should get Config"); @@ -546,7 +550,7 @@ pub mod test_util { discard_logger(), redis.connection.clone(), database.pool.clone(), - campaign_remaining + campaign_remaining, ); app diff --git a/sentry/src/main.rs b/sentry/src/main.rs index 634831e5d..0ab25891c 100644 --- a/sentry/src/main.rs +++ b/sentry/src/main.rs @@ -10,7 +10,7 @@ use primitives::adapter::{Adapter, DummyAdapterOptions, KeystoreOptions}; use primitives::config::configuration; use primitives::util::tests::prep_db::{AUTH, IDS}; use primitives::ValidatorId; -use sentry::db::{CampaignRemaining, postgres_connection, redis_connection, setup_migrations}; +use sentry::db::{postgres_connection, redis_connection, setup_migrations, CampaignRemaining}; use sentry::Application; use slog::{error, info, Logger}; use std::{ @@ -120,14 +120,28 @@ async fn main() -> Result<(), Box> { match adapter { AdapterTypes::EthereumAdapter(adapter) => { run( - Application::new(*adapter, config, logger, redis, postgres, campaign_remaining), + Application::new( + *adapter, + config, + logger, + redis, + postgres, + campaign_remaining, + ), socket_addr, ) .await } AdapterTypes::DummyAdapter(adapter) => { run( - Application::new(*adapter, config, logger, redis, postgres, campaign_remaining), + Application::new( + *adapter, + config, + logger, + redis, + postgres, + campaign_remaining, + ), socket_addr, ) .await diff --git a/sentry/src/payout.rs b/sentry/src/payout.rs index 3a805e119..9c95751a5 100644 --- a/sentry/src/payout.rs +++ b/sentry/src/payout.rs @@ -77,9 +77,7 @@ pub fn get_payout( if output.show { let price = match output.price.get(&event_type) { - Some(output_price) => { - max(pricing.min, min(pricing.max, *output_price)) - } + Some(output_price) => max(pricing.min, min(pricing.max, *output_price)), None => max(pricing.min, pricing.max), }; diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index bf68cfdd7..955e6ab6c 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -1,38 +1,22 @@ -<<<<<<< HEAD -use crate::{success_response, Application, ResponseError}; -use hyper::{Body, Request, Response}; -use primitives::{ - adapter::Adapter, - sentry::{campaign_create::CreateCampaign, SuccessResponse}, -======= use crate::{ - access::{self, check_access}, db::{ - accounting::get_accounting_spent, + accounting::{get_accounting, Side}, campaign::{get_campaigns_by_channel, insert_campaign, update_campaign}, spendable::fetch_spendable, CampaignRemaining, DbPool, RedisError, }, - success_response, Application, Auth, ResponseError, Session, + success_response, Application, Auth, ResponseError, }; -use chrono::Utc; use deadpool_postgres::PoolError; use hyper::{Body, Request, Response}; use primitives::{ adapter::Adapter, campaign_validator::Validator, - sentry::{ - campaign_create::{CreateCampaign, ModifyCampaign}, - Event, SuccessResponse, - }, + sentry::campaign_create::{CreateCampaign, ModifyCampaign}, Address, Campaign, UnifiedNum, }; use slog::error; -use std::{ - cmp::{max, Ordering}, - collections::HashMap, ->>>>>>> issue-382-campaign-routes -}; +use std::cmp::{max, Ordering}; use thiserror::Error; use tokio_postgres::error::SqlState; @@ -88,9 +72,15 @@ pub async fn create_campaign( let total_remaining = { - let accounting_spent = - get_accounting_spent(app.pool.clone(), &campaign.creator, &campaign.channel.id()) - .await?; + let accounting_spent = get_accounting( + app.pool.clone(), + campaign.channel.id(), + campaign.creator, + Side::Spender, + ) + .await? + .map(|accounting| accounting.amount) + .unwrap_or_default(); let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()) @@ -168,7 +158,7 @@ pub async fn create_campaign( } pub mod update_campaign { - use crate::db::CampaignRemaining; + use crate::db::{accounting::Side, CampaignRemaining}; use super::*; @@ -221,9 +211,15 @@ pub mod update_campaign { // sum(AllChannelCampaigns.map(getRemaining)) + DeltaBudgetForMutatedCampaign <= totalDeposited - totalSpent // sum(AllChannelCampaigns.map(getRemaining)) - DeltaBudgetForMutatedCampaign <= totalDeposited - totalSpent if let Some(delta_budget) = delta_budget { - let accounting_spent = - get_accounting_spent(pool.clone(), &campaign.creator, &campaign.channel.id()) - .await?; + let accounting_spent = get_accounting( + pool.clone(), + campaign.channel.id(), + campaign.creator, + Side::Spender, + ) + .await? + .map(|accounting| accounting.amount) + .unwrap_or_default(); let latest_spendable = fetch_spendable(pool.clone(), &campaign.creator, &campaign.channel.id()) @@ -779,12 +775,11 @@ pub mod insert_events { mod test { use super::{update_campaign::modify_campaign, *}; use crate::{ - db::{accounting::insert_accounting, spendable::insert_spendable}, + db::{accounting::update_accounting, spendable::insert_spendable}, test_util::setup_dummy_app, }; use hyper::StatusCode; use primitives::{ - sentry::accounting::{Balances, CheckedState}, spender::{Deposit, Spendable}, util::tests::prep_db::DUMMY_CAMPAIGN, ValidatorId, @@ -831,13 +826,15 @@ mod test { .await .expect("Should insert Spendable for Campaign creator")); - let mut balances = Balances::::default(); - balances.add_spender(create.creator); - - // TODO: Replace this once https://github.com/AdExNetwork/adex-validator-stack-rust/pull/413 is merged - let _accounting = insert_accounting(app.pool.clone(), create.channel.clone(), balances) - .await - .expect("Should create Accounting"); + let _accounting = update_accounting( + app.pool.clone(), + create.channel.id(), + create.creator, + Side::Spender, + UnifiedNum::default(), + ) + .await + .expect("Should create Accounting"); let create_response = create_campaign(build_request(create), &app) .await @@ -928,7 +925,12 @@ mod test { .await .expect_err("Should return Error response"); - assert_eq!(ResponseError::BadRequest("Not enough deposit left for the new campaign's budget".to_string()), create_err); + assert_eq!( + ResponseError::BadRequest( + "Not enough deposit left for the new campaign's budget".to_string() + ), + create_err + ); } // modify first campaign, by lowering the budget from 1000 to 900 @@ -967,7 +969,7 @@ mod test { .await .expect("Should return create campaign"); - let json = hyper::body::to_bytes(create_response.into_body()) + let json = hyper::body::to_bytes(create_response.into_body()) .await .expect("Should get json"); @@ -991,12 +993,13 @@ mod test { targeting_rules: None, }; - let modify_err = - modify_campaign(&app.pool, &app.campaign_remaining, modified, modify) - .await - .expect_err("Should return Error response"); + let modify_err = modify_campaign(&app.pool, &app.campaign_remaining, modified, modify) + .await + .expect_err("Should return Error response"); - assert!(matches!(modify_err, Error::NewBudget(string) if string == "Not enough deposit left for the campaign's new budget")); + assert!( + matches!(modify_err, Error::NewBudget(string) if string == "Not enough deposit left for the campaign's new budget") + ); } } } From 1fca9ec2e3eadbb18e44c80dca31ebfb8c64c355 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 10 Aug 2021 09:05:16 +0300 Subject: [PATCH 47/49] fix clippy warnings --- adapter/src/dummy.rs | 4 +- adapter/src/ethereum.rs | 14 +++--- adview-manager/src/lib.rs | 2 +- primitives/src/adapter.rs | 2 +- primitives/src/big_num.rs | 2 +- primitives/src/campaign.rs | 2 +- primitives/src/campaign_validator.rs | 2 +- primitives/src/channel.rs | 6 +-- primitives/src/sentry.rs | 2 +- primitives/src/sentry/accounting.rs | 12 ++++-- primitives/src/util/tests/time.rs | 2 +- primitives/src/validator.rs | 4 +- sentry/src/access.rs | 10 ++--- sentry/src/analytics_recorder.rs | 1 - sentry/src/db.rs | 2 +- sentry/src/lib.rs | 54 ++++++++++-------------- sentry/src/payout.rs | 2 +- sentry/src/routes/analytics.rs | 4 +- sentry/src/routes/campaign.rs | 27 ++++++------ sentry/src/routes/channel.rs | 6 +-- sentry/src/routes/validator_message.rs | 2 +- validator_worker/src/core/events.rs | 4 +- validator_worker/src/follower.rs | 16 +++---- validator_worker/src/heartbeat.rs | 2 +- validator_worker/src/leader.rs | 8 ++-- validator_worker/src/main.rs | 14 +++--- validator_worker/src/sentry_interface.rs | 16 +++---- 27 files changed, 109 insertions(+), 113 deletions(-) diff --git a/adapter/src/dummy.rs b/adapter/src/dummy.rs index cc9c98b3a..ef5152332 100644 --- a/adapter/src/dummy.rs +++ b/adapter/src/dummy.rs @@ -92,8 +92,8 @@ impl Adapter for DummyAdapter { Ok(()) } - fn whoami(&self) -> &ValidatorId { - &self.identity + fn whoami(&self) -> ValidatorId { + self.identity } fn sign(&self, state_root: &str) -> AdapterResult { diff --git a/adapter/src/ethereum.rs b/adapter/src/ethereum.rs index 681f58041..cab922829 100644 --- a/adapter/src/ethereum.rs +++ b/adapter/src/ethereum.rs @@ -149,8 +149,8 @@ impl Adapter for EthereumAdapter { Ok(()) } - fn whoami(&self) -> &ValidatorId { - &self.address + fn whoami(&self) -> ValidatorId { + self.address } fn sign(&self, state_root: &str) -> AdapterResult { @@ -263,7 +263,7 @@ impl Adapter for EthereumAdapter { address: self.whoami().to_checksum(), }; - ewt_sign(&wallet, &self.keystore_pwd, &payload) + ewt_sign(wallet, &self.keystore_pwd, &payload) .map_err(|err| AdapterError::Adapter(Error::SignMessage(err).into())) } @@ -401,8 +401,8 @@ fn hash_message(message: &[u8]) -> [u8; 32] { let message_length = message.len(); let mut result = Keccak::new_keccak256(); - result.update(&format!("{}{}", eth, message_length).as_bytes()); - result.update(&message); + result.update(format!("{}{}", eth, message_length).as_bytes()); + result.update(message); let mut res: [u8; 32] = [0; 32]; result.finalize(&mut res); @@ -453,7 +453,7 @@ pub fn ewt_sign( base64::URL_SAFE_NO_PAD, ); let message = Message::from(hash_message( - &format!("{}.{}", header_encoded, payload_encoded).as_bytes(), + format!("{}.{}", header_encoded, payload_encoded).as_bytes(), )); let signature: Signature = signer .sign(password, &message) @@ -475,7 +475,7 @@ pub fn ewt_verify( token: &str, ) -> Result { let message = Message::from(hash_message( - &format!("{}.{}", header_encoded, payload_encoded).as_bytes(), + format!("{}.{}", header_encoded, payload_encoded).as_bytes(), )); let decoded_signature = base64::decode_config(&token, base64::URL_SAFE_NO_PAD) diff --git a/adview-manager/src/lib.rs b/adview-manager/src/lib.rs index cae144279..bc845304d 100644 --- a/adview-manager/src/lib.rs +++ b/adview-manager/src/lib.rs @@ -172,7 +172,7 @@ fn get_unit_html( ) -> String { let image_url = normalize_url(&ad_unit.media_url); - let element_html = if is_video(&ad_unit) { + let element_html = if is_video(ad_unit) { video_html(on_load, size, &image_url, &ad_unit.media_mime) } else { image_html(on_load, size, &image_url) diff --git a/primitives/src/adapter.rs b/primitives/src/adapter.rs index af336daff..8eb0ed347 100644 --- a/primitives/src/adapter.rs +++ b/primitives/src/adapter.rs @@ -81,7 +81,7 @@ pub trait Adapter: Send + Sync + fmt::Debug + Clone { fn unlock(&mut self) -> AdapterResult<(), Self::AdapterError>; /// Get Adapter whoami - fn whoami(&self) -> &ValidatorId; + fn whoami(&self) -> ValidatorId; /// Signs the provided state_root fn sign(&self, state_root: &str) -> AdapterResult; diff --git a/primitives/src/big_num.rs b/primitives/src/big_num.rs index 2fd30bbb6..0633baea9 100644 --- a/primitives/src/big_num.rs +++ b/primitives/src/big_num.rs @@ -243,7 +243,7 @@ impl TryFrom<&str> for BigNum { type Error = super::DomainError; fn try_from(num: &str) -> Result { - let big_uint = BigUint::from_str(&num) + let big_uint = BigUint::from_str(num) .map_err(|err| super::DomainError::InvalidArgument(err.to_string()))?; Ok(Self(big_uint)) diff --git a/primitives/src/campaign.rs b/primitives/src/campaign.rs index ad3ca0167..7a427689d 100644 --- a/primitives/src/campaign.rs +++ b/primitives/src/campaign.rs @@ -318,7 +318,7 @@ pub mod validators { } pub fn iter(&self) -> Iter<'_> { - Iter::new(&self) + Iter::new(self) } } diff --git a/primitives/src/campaign_validator.rs b/primitives/src/campaign_validator.rs index 657b12040..d807aca3c 100644 --- a/primitives/src/campaign_validator.rs +++ b/primitives/src/campaign_validator.rs @@ -57,7 +57,7 @@ impl Validator for Campaign { return Err(Validation::UnlistedValidator.into()); } - if !creator_listed(&self, &config.creators_whitelist) { + if !creator_listed(self, &config.creators_whitelist) { return Err(Validation::UnlistedCreator.into()); } diff --git a/primitives/src/channel.rs b/primitives/src/channel.rs index 507d4060b..84ee2b0ad 100644 --- a/primitives/src/channel.rs +++ b/primitives/src/channel.rs @@ -238,9 +238,9 @@ impl SpecValidators { pub fn find(&self, validator_id: &ValidatorId) -> Option> { if &self.leader().id == validator_id { - Some(SpecValidator::Leader(&self.leader())) + Some(SpecValidator::Leader(self.leader())) } else if &self.follower().id == validator_id { - Some(SpecValidator::Follower(&self.follower())) + Some(SpecValidator::Follower(self.follower())) } else { None } @@ -257,7 +257,7 @@ impl SpecValidators { } pub fn iter(&self) -> Iter<'_> { - Iter::new(&self) + Iter::new(self) } } diff --git a/primitives/src/sentry.rs b/primitives/src/sentry.rs index ba07b4035..cda0813b9 100644 --- a/primitives/src/sentry.rs +++ b/primitives/src/sentry.rs @@ -380,7 +380,7 @@ pub mod campaign_create { } } } - + // All editable fields stored in one place, used for checking when a budget is changed #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ModifyCampaign { diff --git a/primitives/src/sentry/accounting.rs b/primitives/src/sentry/accounting.rs index 74a3aa677..53a4efb44 100644 --- a/primitives/src/sentry/accounting.rs +++ b/primitives/src/sentry/accounting.rs @@ -69,14 +69,18 @@ impl Balances { Ok(()) } - /// Adds the spender to the Balances with `UnifiedNum::from(0)` if he does not exist + /// Adds the spender to the Balances with `0` if he does not exist pub fn add_spender(&mut self, spender: Address) { - self.spenders.entry(spender).or_insert(UnifiedNum::from(0)); + self.spenders + .entry(spender) + .or_insert_with(UnifiedNum::default); } - /// Adds the earner to the Balances with `UnifiedNum::from(0)` if he does not exist + /// Adds the earner to the Balances with `0` if he does not exist pub fn add_earner(&mut self, earner: Address) { - self.earners.entry(earner).or_insert(UnifiedNum::from(0)); + self.earners + .entry(earner) + .or_insert_with(UnifiedNum::default); } } diff --git a/primitives/src/util/tests/time.rs b/primitives/src/util/tests/time.rs index 349165fae..fc16e5422 100644 --- a/primitives/src/util/tests/time.rs +++ b/primitives/src/util/tests/time.rs @@ -25,5 +25,5 @@ pub fn past_datetime(from: Option<&DateTime>) -> DateTime { let from = from.unwrap_or(&default_from); - datetime_between(&from, Some(&to)) + datetime_between(from, Some(&to)) } diff --git a/primitives/src/validator.rs b/primitives/src/validator.rs index 72b89f7a1..9403c45a7 100644 --- a/primitives/src/validator.rs +++ b/primitives/src/validator.rs @@ -27,7 +27,7 @@ impl ValidatorId { } pub fn inner(&self) -> &[u8; 20] { - &self.0.as_bytes() + self.0.as_bytes() } } @@ -53,7 +53,7 @@ impl From<&[u8; 20]> for ValidatorId { impl AsRef<[u8]> for ValidatorId { fn as_ref(&self) -> &[u8] { - &self.0.as_ref() + self.0.as_ref() } } diff --git a/sentry/src/access.rs b/sentry/src/access.rs index a3fca1578..518fce44c 100644 --- a/sentry/src/access.rs +++ b/sentry/src/access.rs @@ -40,7 +40,7 @@ pub async fn check_access( let auth_uid = auth.map(|auth| auth.uid.to_string()).unwrap_or_default(); // Rules for events - if forbidden_country(&session) || forbidden_referrer(&session) { + if forbidden_country(session) || forbidden_referrer(session) { return Err(Error::ForbiddenReferrer); } @@ -79,11 +79,11 @@ pub async fn check_access( let apply_all_rules = try_join_all(rules.iter().map(|rule| { apply_rule( redis.clone(), - &rule, - &events, - &campaign, + rule, + events, + campaign, &auth_uid, - &session, + session, ) })); diff --git a/sentry/src/analytics_recorder.rs b/sentry/src/analytics_recorder.rs index a7e3b2cfc..3d2d07d3c 100644 --- a/sentry/src/analytics_recorder.rs +++ b/sentry/src/analytics_recorder.rs @@ -114,7 +114,6 @@ pub async fn record( .ignore(); } } - _ => {} }); if let Err(err) = db.query_async::<_, Option>(&mut conn).await { diff --git a/sentry/src/db.rs b/sentry/src/db.rs index 63578cac3..336a986f5 100644 --- a/sentry/src/db.rs +++ b/sentry/src/db.rs @@ -77,7 +77,7 @@ pub async fn setup_migrations(environment: &str) { .database_password(POSTGRES_PASSWORD.as_str()) .database_host(POSTGRES_HOST.as_str()) .database_port(*POSTGRES_PORT) - .database_name(&POSTGRES_DB.as_ref().unwrap_or(&POSTGRES_USER)) + .database_name(POSTGRES_DB.as_ref().unwrap_or(&POSTGRES_USER)) .build() .expect("Should build migration settings"); diff --git a/sentry/src/lib.rs b/sentry/src/lib.rs index 713aa80e7..b8313ed5d 100644 --- a/sentry/src/lib.rs +++ b/sentry/src/lib.rs @@ -125,63 +125,63 @@ impl Application { None => Default::default(), }; - let req = match Authenticate.call(req, &self).await { + let req = match Authenticate.call(req, self).await { Ok(req) => req, Err(error) => return map_response_error(error), }; let mut response = match (req.uri().path(), req.method()) { - ("/cfg", &Method::GET) => config(req, &self).await, - ("/channel", &Method::POST) => create_channel(req, &self).await, - ("/channel/list", &Method::GET) => channel_list(req, &self).await, - ("/channel/validate", &Method::POST) => channel_validate(req, &self).await, + ("/cfg", &Method::GET) => config(req, self).await, + ("/channel", &Method::POST) => create_channel(req, self).await, + ("/channel/list", &Method::GET) => channel_list(req, self).await, + ("/channel/validate", &Method::POST) => channel_validate(req, self).await, - ("/analytics", &Method::GET) => analytics(req, &self).await, + ("/analytics", &Method::GET) => analytics(req, self).await, ("/analytics/advanced", &Method::GET) => { - let req = match AuthRequired.call(req, &self).await { + let req = match AuthRequired.call(req, self).await { Ok(req) => req, Err(error) => { return map_response_error(error); } }; - advanced_analytics(req, &self).await + advanced_analytics(req, self).await } ("/analytics/for-advertiser", &Method::GET) => { - let req = match AuthRequired.call(req, &self).await { + let req = match AuthRequired.call(req, self).await { Ok(req) => req, Err(error) => { return map_response_error(error); } }; - advertiser_analytics(req, &self).await + advertiser_analytics(req, self).await } ("/analytics/for-publisher", &Method::GET) => { - let req = match AuthRequired.call(req, &self).await { + let req = match AuthRequired.call(req, self).await { Ok(req) => req, Err(error) => { return map_response_error(error); } }; - publisher_analytics(req, &self).await + publisher_analytics(req, self).await } // For creating campaigns ("/v5/campaign", &Method::POST) => { - let req = match AuthRequired.call(req, &self).await { + let req = match AuthRequired.call(req, self).await { Ok(req) => req, Err(error) => { return map_response_error(error); } }; - create_campaign(req, &self).await + create_campaign(req, self).await } - (route, _) if route.starts_with("/analytics") => analytics_router(req, &self).await, + (route, _) if route.starts_with("/analytics") => analytics_router(req, self).await, // This is important because it prevents us from doing // expensive regex matching for routes without /channel - (path, _) if path.starts_with("/channel") => channels_router(req, &self).await, - (path, _) if path.starts_with("/v5/campaign") => campaigns_router(req, &self).await, + (path, _) if path.starts_with("/channel") => channels_router(req, self).await, + (path, _) if path.starts_with("/v5/campaign") => campaigns_router(req, self).await, _ => Err(ResponseError::NotFound), } .unwrap_or_else(map_response_error); @@ -198,12 +198,12 @@ async fn campaigns_router( ) -> Result, ResponseError> { let (path, method) = (req.uri().path(), req.method()); - if let (Some(_caps), &Method::POST) = (CAMPAIGN_UPDATE_BY_ID.captures(&path), method) { + if let (Some(_caps), &Method::POST) = (CAMPAIGN_UPDATE_BY_ID.captures(path), method) { let req = CampaignLoad.call(req, app).await?; update_campaign::handle_route(req, app).await } else if let (Some(caps), &Method::POST) = - (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(&path), method) + (INSERT_EVENTS_BY_CAMPAIGN_ID.captures(path), method) { let param = RouteParams(vec![caps .get(1) @@ -214,7 +214,7 @@ async fn campaigns_router( campaign::insert_events::handle_route(req, app).await } else if let (Some(_caps), &Method::POST) = - (CLOSE_CAMPAIGN_BY_CAMPAIGN_ID.captures(&path), method) + (CLOSE_CAMPAIGN_BY_CAMPAIGN_ID.captures(path), method) { // TODO AIP#61: Close campaign: // - only by creator @@ -308,16 +308,6 @@ async fn channels_router( ) -> Result, ResponseError> { let (path, method) = (req.uri().path().to_owned(), req.method()); - // regex matching for routes with params - /* if let (Some(caps), &Method::POST) = (CREATE_EVENTS_BY_CHANNEL_ID.captures(&path), method) { - let param = RouteParams(vec![caps - .get(1) - .map_or("".to_string(), |m| m.as_str().to_string())]); - - req.extensions_mut().insert(param); - - insert_events(req, app).await - } else */ if let (Some(caps), &Method::GET) = (LAST_APPROVED_BY_CHANNEL_ID.captures(&path), method) { let param = RouteParams(vec![caps .get(1) @@ -353,7 +343,7 @@ async fn channels_router( } }; - list_validator_messages(req, &app, &extract_params.0, &extract_params.1).await + list_validator_messages(req, app, &extract_params.0, &extract_params.1).await } else if let (Some(caps), &Method::POST) = (CHANNEL_VALIDATOR_MESSAGES.captures(&path), method) { let param = RouteParams(vec![caps @@ -368,7 +358,7 @@ async fn channels_router( .apply(req, app) .await?; - create_validator_messages(req, &app).await + create_validator_messages(req, app).await } else if let (Some(caps), &Method::GET) = (CHANNEL_EVENTS_AGGREGATES.captures(&path), method) { req = AuthRequired.call(req, app).await?; diff --git a/sentry/src/payout.rs b/sentry/src/payout.rs index 9c95751a5..7226b77b9 100644 --- a/sentry/src/payout.rs +++ b/sentry/src/payout.rs @@ -35,7 +35,7 @@ pub fn get_payout( } => { let targeting_rules = campaign.targeting_rules.clone(); - let pricing = get_pricing_bounds(&campaign, &event_type); + let pricing = get_pricing_bounds(campaign, &event_type); if targeting_rules.is_empty() { Ok(Some((*publisher, pricing.min))) diff --git a/sentry/src/routes/analytics.rs b/sentry/src/routes/analytics.rs index c0c3de4ed..cfb7a72a6 100644 --- a/sentry/src/routes/analytics.rs +++ b/sentry/src/routes/analytics.rs @@ -80,7 +80,7 @@ pub async fn process_analytics( app: &Application, analytics_type: AnalyticsType, ) -> Result { - let query = serde_urlencoded::from_str::(&req.uri().query().unwrap_or(""))?; + let query = serde_urlencoded::from_str::(req.uri().query().unwrap_or(""))?; query .is_valid() .map_err(|e| ResponseError::BadRequest(e.to_string()))?; @@ -113,7 +113,7 @@ pub async fn advanced_analytics( let auth = req.extensions().get::().expect("auth is required"); let advertiser_channels = advertiser_channel_ids(&app.pool, &auth.uid).await?; - let query = serde_urlencoded::from_str::(&req.uri().query().unwrap_or(""))?; + let query = serde_urlencoded::from_str::(req.uri().query().unwrap_or(""))?; let response = get_advanced_reports( &app.redis, diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 955e6ab6c..33a307f15 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -85,13 +85,13 @@ pub async fn create_campaign( let latest_spendable = fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()) .await? - .ok_or(ResponseError::BadRequest( + .ok_or_else(|| ResponseError::BadRequest( "No spendable amount found for the Campaign creator".to_string(), ))?; // Gets the latest Spendable for this (spender, channelId) pair let total_deposited = latest_spendable.deposit.total; - total_deposited.checked_sub(&accounting_spent).ok_or( + total_deposited.checked_sub(&accounting_spent).ok_or_else(|| ResponseError::FailedValidation("No more budget remaining".to_string()), )? }; @@ -113,7 +113,9 @@ pub async fn create_campaign( .checked_add(&campaign.budget) .ok_or(Error::Calculation)?; - if !(campaigns_remaining_sum <= total_remaining) || campaign.budget > total_remaining { + // `new_campaigns_remaining <= total_remaining` should be upheld + // `campaign.budget < total_remaining` should also be upheld! + if campaigns_remaining_sum > total_remaining || campaign.budget > total_remaining { return Err(ResponseError::BadRequest( "Not enough deposit left for the new campaign's budget".to_string(), )); @@ -232,7 +234,7 @@ pub mod update_campaign { let total_remaining = total_deposited .checked_sub(&accounting_spent) .ok_or(Error::Calculation)?; - let channel_campaigns = get_campaigns_by_channel(&pool, &campaign.channel.id()) + let channel_campaigns = get_campaigns_by_channel(pool, &campaign.channel.id()) .await? .iter() .map(|c| c.id) @@ -257,7 +259,8 @@ pub mod update_campaign { } .ok_or(Error::Calculation)?; - if !(new_campaigns_remaining <= total_remaining) { + // `new_campaigns_remaining <= total_remaining` should be upheld + if new_campaigns_remaining > total_remaining { return Err(Error::NewBudget( "Not enough deposit left for the campaign's new budget".to_string(), )); @@ -280,7 +283,7 @@ pub mod update_campaign { } let modified_campaign = modify_campaign.apply(campaign); - update_campaign(&pool, &modified_campaign).await?; + update_campaign(pool, &modified_campaign).await?; Ok(modified_campaign) } @@ -312,7 +315,7 @@ pub mod update_campaign { .get_remaining_opt(campaign.id) .await? .map(|remaining| UnifiedNum::from(max(0, remaining).unsigned_abs())) - .ok_or(Error::FailedUpdate( + .ok_or_else(|| Error::FailedUpdate( "No remaining entry for campaign".to_string(), ))?; @@ -349,7 +352,7 @@ pub mod update_campaign { .ok_or(Error::Calculation)?; // old remaining > new remaining let decrease_by = old_remaining - .checked_sub(&new_remaining) + .checked_sub(new_remaining) .ok_or(Error::Calculation)?; DeltaBudget::Decrease(decrease_by) @@ -457,7 +460,7 @@ pub mod insert_events { session, auth, &app.config.ip_rate_limit, - &campaign, + campaign, &events, ) .await @@ -489,7 +492,7 @@ pub mod insert_events { Some((earner, payout)) => spend_for_event( &app.pool, app.redis.clone(), - &campaign, + campaign, earner, leader, follower, @@ -520,9 +523,9 @@ pub mod insert_events { ) -> Result<(), Error> { // distribute fees let leader_fee = - calculate_fee((earner, amount), &leader).map_err(EventError::FeeCalculation)?; + calculate_fee((earner, amount), leader).map_err(EventError::FeeCalculation)?; let follower_fee = - calculate_fee((earner, amount), &follower).map_err(EventError::FeeCalculation)?; + calculate_fee((earner, amount), follower).map_err(EventError::FeeCalculation)?; // First update redis `campaignRemaining:{CampaignId}` key let spending = [amount, leader_fee, follower_fee] diff --git a/sentry/src/routes/channel.rs b/sentry/src/routes/channel.rs index ca337b6f7..f15c717b7 100644 --- a/sentry/src/routes/channel.rs +++ b/sentry/src/routes/channel.rs @@ -82,7 +82,7 @@ pub async fn channel_list( req: Request, app: &Application, ) -> Result, ResponseError> { - let query = serde_urlencoded::from_str::(&req.uri().query().unwrap_or(""))?; + let query = serde_urlencoded::from_str::(req.uri().query().unwrap_or(""))?; let skip = query .page .checked_mul(app.config.channels_find_limit.into()) @@ -148,7 +148,7 @@ pub async fn last_approved( return Ok(default_response); } - let query = serde_urlencoded::from_str::(&req.uri().query().unwrap_or(""))?; + let query = serde_urlencoded::from_str::(req.uri().query().unwrap_or(""))?; let validators = channel.spec.validators; let channel_id = channel.id; let heartbeats = if query.with_heartbeat.is_some() { @@ -212,7 +212,7 @@ pub async fn create_validator_messages( None => Err(ResponseError::Unauthorized), _ => { try_join_all(messages.iter().map(|message| { - insert_validator_messages(&app.pool, &channel, &session.uid, &message) + insert_validator_messages(&app.pool, &channel, &session.uid, message) })) .await?; diff --git a/sentry/src/routes/validator_message.rs b/sentry/src/routes/validator_message.rs index 3c2fc375d..e4c98d71b 100644 --- a/sentry/src/routes/validator_message.rs +++ b/sentry/src/routes/validator_message.rs @@ -46,7 +46,7 @@ pub async fn list_validator_messages( message_types: &[String], ) -> Result, ResponseError> { let query = - serde_urlencoded::from_str::(&req.uri().query().unwrap_or(""))?; + serde_urlencoded::from_str::(req.uri().query().unwrap_or(""))?; let channel = req .extensions() diff --git a/validator_worker/src/core/events.rs b/validator_worker/src/core/events.rs index 211a99527..6912f1e17 100644 --- a/validator_worker/src/core/events.rs +++ b/validator_worker/src/core/events.rs @@ -59,9 +59,9 @@ fn _merge_payouts_into_balances<'a, T: Iterator>( let new_balance = new_balances.entry(*acc).or_insert_with(|| 0.into()); - *new_balance += &to_add; + *new_balance += to_add; - remaining = remaining.checked_sub(&to_add).ok_or_else(|| { + remaining = remaining.checked_sub(to_add).ok_or_else(|| { DomainError::RuleViolation("remaining must never be negative".to_string()) })?; } diff --git a/validator_worker/src/follower.rs b/validator_worker/src/follower.rs index 7d200deeb..2e480a732 100644 --- a/validator_worker/src/follower.rs +++ b/validator_worker/src/follower.rs @@ -77,7 +77,7 @@ pub async fn tick( _ => false, }; - let producer_tick = producer::tick(&iface).await?; + let producer_tick = producer::tick(iface).await?; let empty_balances = BalancesMap::default(); let balances = match &producer_tick { producer::TickStatus::Sent { new_accounting, .. } => &new_accounting.balances, @@ -85,13 +85,13 @@ pub async fn tick( producer::TickStatus::EmptyBalances => &empty_balances, }; let approve_state_result = if let (Some(new_state), false) = (new_msg, latest_is_responded_to) { - on_new_state(&iface, &balances, &new_state).await? + on_new_state(iface, balances, &new_state).await? } else { ApproveStateResult::Sent(None) }; Ok(TickStatus { - heartbeat: heartbeat(&iface, &balances).await?, + heartbeat: heartbeat(iface, balances).await?, approve_state: approve_state_result, producer_tick, }) @@ -104,8 +104,8 @@ async fn on_new_state<'a, A: Adapter + 'static>( ) -> Result, Box> { let proposed_balances = new_state.balances.clone(); let proposed_state_root = new_state.state_root.clone(); - if proposed_state_root != hex::encode(get_state_root_hash(&iface, &proposed_balances)?) { - return Ok(on_error(&iface, &new_state, InvalidNewState::RootHash).await); + if proposed_state_root != hex::encode(get_state_root_hash(iface, &proposed_balances)?) { + return Ok(on_error(iface, new_state, InvalidNewState::RootHash).await); } if !iface.adapter.verify( @@ -113,7 +113,7 @@ async fn on_new_state<'a, A: Adapter + 'static>( &proposed_state_root, &new_state.signature, )? { - return Ok(on_error(&iface, &new_state, InvalidNewState::Signature).await); + return Ok(on_error(iface, new_state, InvalidNewState::Signature).await); } let last_approve_response = iface.get_last_approved().await?; @@ -126,12 +126,12 @@ async fn on_new_state<'a, A: Adapter + 'static>( }; if !is_valid_transition(&iface.channel, &prev_balances, &proposed_balances) { - return Ok(on_error(&iface, &new_state, InvalidNewState::Transition).await); + return Ok(on_error(iface, new_state, InvalidNewState::Transition).await); } let health = get_health(&iface.channel, balances, &proposed_balances); if health < u64::from(iface.config.health_unsignable_promilles) { - return Ok(on_error(&iface, &new_state, InvalidNewState::Health).await); + return Ok(on_error(iface, new_state, InvalidNewState::Health).await); } let signature = iface.adapter.sign(&new_state.state_root)?; diff --git a/validator_worker/src/heartbeat.rs b/validator_worker/src/heartbeat.rs index 35af963dc..4e2016558 100644 --- a/validator_worker/src/heartbeat.rs +++ b/validator_worker/src/heartbeat.rs @@ -55,7 +55,7 @@ pub async fn heartbeat( }); if should_send { - Ok(Some(send_heartbeat(&iface).await?)) + Ok(Some(send_heartbeat(iface).await?)) } else { Ok(None) } diff --git a/validator_worker/src/leader.rs b/validator_worker/src/leader.rs index 792764b8c..7b582b7d4 100644 --- a/validator_worker/src/leader.rs +++ b/validator_worker/src/leader.rs @@ -21,11 +21,11 @@ pub struct TickStatus { pub async fn tick( iface: &SentryApi, ) -> Result, Box> { - let producer_tick = producer::tick(&iface).await?; + let producer_tick = producer::tick(iface).await?; let empty_balances = BalancesMap::default(); let (balances, new_state) = match &producer_tick { producer::TickStatus::Sent { new_accounting, .. } => { - let new_state = on_new_accounting(&iface, new_accounting).await?; + let new_state = on_new_accounting(iface, new_accounting).await?; (&new_accounting.balances, Some(new_state)) } producer::TickStatus::NoNewEventAggr(balances) => (balances, None), @@ -33,7 +33,7 @@ pub async fn tick( }; Ok(TickStatus { - heartbeat: heartbeat(&iface, &balances).await?, + heartbeat: heartbeat(iface, balances).await?, new_state, producer_tick, }) @@ -43,7 +43,7 @@ async fn on_new_accounting( iface: &SentryApi, new_accounting: &Accounting, ) -> Result>, Box> { - let state_root_raw = get_state_root_hash(&iface, &new_accounting.balances)?; + let state_root_raw = get_state_root_hash(iface, &new_accounting.balances)?; let state_root = hex::encode(state_root_raw); let signature = iface.adapter.sign(&state_root)?; diff --git a/validator_worker/src/main.rs b/validator_worker/src/main.rs index 3cdd6b928..acf42fa37 100644 --- a/validator_worker/src/main.rs +++ b/validator_worker/src/main.rs @@ -116,10 +116,10 @@ fn main() -> Result<(), Box> { match adapter { AdapterTypes::EthereumAdapter(ethadapter) => { - run(is_single_tick, &sentry_url, &config, *ethadapter, &logger) + run(is_single_tick, sentry_url, &config, *ethadapter, &logger) } AdapterTypes::DummyAdapter(dummyadapter) => { - run(is_single_tick, &sentry_url, &config, *dummyadapter, &logger) + run(is_single_tick, sentry_url, &config, *dummyadapter, &logger) } } } @@ -144,9 +144,9 @@ fn run( let rt = Runtime::new()?; if is_single_tick { - rt.block_on(iterate_channels(args, &logger)); + rt.block_on(iterate_channels(args, logger)); } else { - rt.block_on(infinite(args, &logger)); + rt.block_on(infinite(args, logger)); } Ok(()) @@ -161,7 +161,7 @@ async fn infinite(args: Args, logger: &Logger) { } async fn iterate_channels(args: Args, logger: &Logger) { - let result = all_channels(&args.sentry_url, args.adapter.whoami()).await; + let result = all_channels(&args.sentry_url, &args.adapter.whoami()).await; let channels = match result { Ok(channels) => channels, @@ -197,10 +197,10 @@ async fn validator_tick( config: &Config, logger: &Logger, ) -> Result<(ChannelId, Box), ValidatorWorkerError> { - let whoami = *adapter.whoami(); + let whoami = adapter.whoami(); // Cloning the `Logger` is cheap, see documentation for more info - let sentry = SentryApi::init(adapter, channel.clone(), &config, logger.clone()) + let sentry = SentryApi::init(adapter, channel.clone(), config, logger.clone()) .map_err(ValidatorWorkerError::SentryApi)?; let duration = Duration::from_millis(config.validator_tick_timeout as u64); diff --git a/validator_worker/src/sentry_interface.rs b/validator_worker/src/sentry_interface.rs index bf6be3fc1..ca8e8b61f 100644 --- a/validator_worker/src/sentry_interface.rs +++ b/validator_worker/src/sentry_interface.rs @@ -87,7 +87,7 @@ impl SentryApi { .map_err(Error::BuildingClient)?; // validate that we are to validate the channel - match channel.spec.validators.find(adapter.whoami()) { + match channel.spec.validators.find(&adapter.whoami()) { Some(ref spec_validator) => { let validator = spec_validator.validator(); let validator_url = format!("{}/channel/{}", validator.url, channel.id); @@ -117,7 +117,7 @@ impl SentryApi { None => Err(Error::MissingWhoamiInChannelValidators { channel: channel.id, validators: channel.spec.validators.iter().map(|v| v.id).collect(), - whoami: *adapter.whoami(), + whoami: adapter.whoami(), }), } } @@ -129,9 +129,9 @@ impl SentryApi { join_all(self.propagate_to.iter().map(|(validator, auth_token)| { propagate_to::( &self.channel.id, - &auth_token, + auth_token, &self.client, - &validator, + validator, messages, ) })) @@ -165,7 +165,7 @@ impl SentryApi { &self, message_types: &[&str], ) -> Result, Error> { - self.get_latest_msg(self.adapter.whoami(), message_types) + self.get_latest_msg(&self.adapter.whoami(), message_types) .await } @@ -196,7 +196,7 @@ impl SentryApi { ) -> Result> { let auth_token = self .adapter - .get_auth(self.adapter.whoami()) + .get_auth(&self.adapter.whoami()) .map_err(Error::ValidatorAuthentication)?; let url = format!( @@ -250,13 +250,13 @@ pub async fn all_channels( whoami: &ValidatorId, ) -> Result, reqwest::Error> { let url = sentry_url.to_owned(); - let first_page = fetch_page(url.clone(), 0, &whoami).await?; + let first_page = fetch_page(url.clone(), 0, whoami).await?; if first_page.total_pages < 2 { Ok(first_page.channels) } else { let all: Vec = - try_join_all((1..first_page.total_pages).map(|i| fetch_page(url.clone(), i, &whoami))) + try_join_all((1..first_page.total_pages).map(|i| fetch_page(url.clone(), i, whoami))) .await?; let result_all: Vec = std::iter::once(first_page) From 05ac2b0fbd50ed22a6599c4ca84a008a62212eea Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 10 Aug 2021 09:10:35 +0300 Subject: [PATCH 48/49] rustfmt --- sentry/src/access.rs | 15 +++------ sentry/src/middleware/campaign.rs | 5 +-- sentry/src/routes/campaign.rs | 53 ++++++++++++++++--------------- 3 files changed, 33 insertions(+), 40 deletions(-) diff --git a/sentry/src/access.rs b/sentry/src/access.rs index 518fce44c..d3435e88d 100644 --- a/sentry/src/access.rs +++ b/sentry/src/access.rs @@ -76,16 +76,11 @@ pub async fn check_access( return Ok(()); } - let apply_all_rules = try_join_all(rules.iter().map(|rule| { - apply_rule( - redis.clone(), - rule, - events, - campaign, - &auth_uid, - session, - ) - })); + let apply_all_rules = try_join_all( + rules + .iter() + .map(|rule| apply_rule(redis.clone(), rule, events, campaign, &auth_uid, session)), + ); apply_all_rules.await.map_err(Error::RulesError).map(|_| ()) } diff --git a/sentry/src/middleware/campaign.rs b/sentry/src/middleware/campaign.rs index 024dc1400..2da58c1b8 100644 --- a/sentry/src/middleware/campaign.rs +++ b/sentry/src/middleware/campaign.rs @@ -37,10 +37,7 @@ impl Middleware for CampaignLoad { #[cfg(test)] mod test { - use primitives::{ - util::tests::prep_db::{DUMMY_CAMPAIGN, IDS}, - Campaign, - }; + use primitives::{util::tests::prep_db::DUMMY_CAMPAIGN, Campaign}; use crate::{db::insert_campaign, test_util::setup_dummy_app}; diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 33a307f15..313c991ef 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -70,31 +70,34 @@ pub async fn create_campaign( let error_response = ResponseError::BadRequest("err occurred; please try again later".to_string()); - let total_remaining = - { - let accounting_spent = get_accounting( - app.pool.clone(), - campaign.channel.id(), - campaign.creator, - Side::Spender, - ) - .await? - .map(|accounting| accounting.amount) - .unwrap_or_default(); + let total_remaining = { + let accounting_spent = get_accounting( + app.pool.clone(), + campaign.channel.id(), + campaign.creator, + Side::Spender, + ) + .await? + .map(|accounting| accounting.amount) + .unwrap_or_default(); - let latest_spendable = - fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()) - .await? - .ok_or_else(|| ResponseError::BadRequest( + let latest_spendable = + fetch_spendable(app.pool.clone(), &campaign.creator, &campaign.channel.id()) + .await? + .ok_or_else(|| { + ResponseError::BadRequest( "No spendable amount found for the Campaign creator".to_string(), - ))?; - // Gets the latest Spendable for this (spender, channelId) pair - let total_deposited = latest_spendable.deposit.total; - - total_deposited.checked_sub(&accounting_spent).ok_or_else(|| - ResponseError::FailedValidation("No more budget remaining".to_string()), - )? - }; + ) + })?; + // Gets the latest Spendable for this (spender, channelId) pair + let total_deposited = latest_spendable.deposit.total; + + total_deposited + .checked_sub(&accounting_spent) + .ok_or_else(|| { + ResponseError::FailedValidation("No more budget remaining".to_string()) + })? + }; let channel_campaigns = get_campaigns_by_channel(&app.pool, &campaign.channel.id()) .await? @@ -315,9 +318,7 @@ pub mod update_campaign { .get_remaining_opt(campaign.id) .await? .map(|remaining| UnifiedNum::from(max(0, remaining).unsigned_abs())) - .ok_or_else(|| Error::FailedUpdate( - "No remaining entry for campaign".to_string(), - ))?; + .ok_or_else(|| Error::FailedUpdate("No remaining entry for campaign".to_string()))?; let campaign_spent = campaign .budget From 598ab52c65be4dd2110ef88f430d731d4babfd9c Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Tue, 10 Aug 2021 09:27:59 +0300 Subject: [PATCH 49/49] sentry - routes - camiang insert events - use CampaignRemaining --- sentry/src/routes/campaign.rs | 87 ++++++++++++----------------------- 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/sentry/src/routes/campaign.rs b/sentry/src/routes/campaign.rs index 313c991ef..4e4b3f653 100644 --- a/sentry/src/routes/campaign.rs +++ b/sentry/src/routes/campaign.rs @@ -370,7 +370,7 @@ pub mod insert_events { use crate::{ access::{self, check_access}, - db::{accounting::spend_amount, DbPool, PoolError, RedisError}, + db::{accounting::spend_amount, CampaignRemaining, DbPool, PoolError, RedisError}, payout::get_payout, spender::fee::calculate_fee, Application, Auth, ResponseError, Session, @@ -384,12 +384,8 @@ pub mod insert_events { }, Address, Campaign, CampaignId, DomainError, UnifiedNum, ValidatorDesc, }; - use redis::aio::MultiplexedConnection; use thiserror::Error; - // TODO AIP#61: Use the Campaign Modify const here - pub const CAMPAIGN_REMAINING_KEY: &str = "campaignRemaining"; - #[derive(Debug, Error)] pub enum Error { #[error(transparent)] @@ -492,7 +488,7 @@ pub mod insert_events { match payout { Some((earner, payout)) => spend_for_event( &app.pool, - app.redis.clone(), + &app.campaign_remaining, campaign, earner, leader, @@ -515,7 +511,7 @@ pub mod insert_events { pub async fn spend_for_event( pool: &DbPool, - mut redis: MultiplexedConnection, + campaign_remaining: &CampaignRemaining, campaign: &Campaign, earner: Address, leader: &ValidatorDesc, @@ -534,14 +530,16 @@ pub mod insert_events { .sum::>() .ok_or(EventError::EventPayoutOverflow)?; - if !has_enough_remaining_budget(&mut redis, campaign.id, spending).await? { + if !has_enough_remaining_budget(campaign_remaining, campaign.id, spending).await? { return Err(Error::Event( EventError::CampaignRemainingNotEnoughForPayout, )); } // The event payout decreases the remaining budget for the Campaign - let remaining = decrease_remaining_budget(&mut redis, campaign.id, spending).await?; + let remaining = campaign_remaining + .decrease_by(campaign.id, spending) + .await?; // Update the Accounting records accordingly let channel_id = campaign.channel.id(); @@ -563,40 +561,22 @@ pub mod insert_events { } async fn has_enough_remaining_budget( - redis: &mut MultiplexedConnection, + campaign_remaining: &CampaignRemaining, campaign: CampaignId, amount: UnifiedNum, ) -> Result { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); - - let remaining = redis::cmd("GET") - .arg(&key) - .query_async::<_, Option>(redis) + let remaining = campaign_remaining + .get_remaining_opt(campaign) .await? .unwrap_or_default(); Ok(remaining > 0 && remaining.unsigned_abs() > amount.to_u64()) } - async fn decrease_remaining_budget( - redis: &mut MultiplexedConnection, - campaign: CampaignId, - amount: UnifiedNum, - ) -> Result { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); - - let remaining = redis::cmd("DECRBY") - .arg(&key) - .arg(amount.to_u64()) - .query_async::<_, i64>(redis) - .await?; - - Ok(remaining) - } - #[cfg(test)] mod test { use primitives::util::tests::prep_db::{ADDRESSES, DUMMY_CAMPAIGN}; + use redis::aio::MultiplexedConnection; use crate::db::{ redis_pool::TESTS_POOL, @@ -605,27 +585,13 @@ pub mod insert_events { use super::*; - /// Helper function to get the Campaign Remaining budget in Redis for the tests - async fn get_campaign_remaining( - redis: &mut MultiplexedConnection, - campaign: CampaignId, - ) -> Option { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); - - redis::cmd("GET") - .arg(&key) - .query_async(redis) - .await - .expect("Should set Campaign remaining key") - } - /// Helper function to set the Campaign Remaining budget in Redis for the tests async fn set_campaign_remaining( redis: &mut MultiplexedConnection, campaign: CampaignId, remaining: i64, ) { - let key = format!("{}:{}", CAMPAIGN_REMAINING_KEY, campaign); + let key = CampaignRemaining::get_key(campaign); redis::cmd("SET") .arg(&key) @@ -638,12 +604,14 @@ pub mod insert_events { #[tokio::test] async fn test_has_enough_remaining_budget() { let mut redis = TESTS_POOL.get().await.expect("Should get redis connection"); + let campaign_remaining = CampaignRemaining::new(redis.connection.clone()); let campaign = DUMMY_CAMPAIGN.id; let amount = UnifiedNum::from(10_000); - let no_remaining_budget_set = has_enough_remaining_budget(&mut redis, campaign, amount) - .await - .expect("Should check campaign remaining"); + let no_remaining_budget_set = + has_enough_remaining_budget(&campaign_remaining, campaign, amount) + .await + .expect("Should check campaign remaining"); assert!( !no_remaining_budget_set, "No remaining budget set, should return false" @@ -652,7 +620,7 @@ pub mod insert_events { set_campaign_remaining(&mut redis, campaign, 9_000).await; let not_enough_remaining_budget = - has_enough_remaining_budget(&mut redis, campaign, amount) + has_enough_remaining_budget(&campaign_remaining, campaign, amount) .await .expect("Should check campaign remaining"); assert!( @@ -663,7 +631,7 @@ pub mod insert_events { set_campaign_remaining(&mut redis, campaign, 11_000).await; let has_enough_remaining_budget = - has_enough_remaining_budget(&mut redis, campaign, amount) + has_enough_remaining_budget(&campaign_remaining, campaign, amount) .await .expect("Should check campaign remaining"); @@ -677,11 +645,13 @@ pub mod insert_events { async fn test_decreasing_remaining_budget() { let mut redis = TESTS_POOL.get().await.expect("Should get redis connection"); let campaign = DUMMY_CAMPAIGN.id; + let campaign_remaining = CampaignRemaining::new(redis.connection.clone()); let amount = UnifiedNum::from(5_000); set_campaign_remaining(&mut redis, campaign, 9_000).await; - let remaining = decrease_remaining_budget(&mut redis, campaign, amount) + let remaining = campaign_remaining + .decrease_by(campaign, amount) .await .expect("Should decrease campaign remaining"); assert_eq!( @@ -689,7 +659,8 @@ pub mod insert_events { "Should decrease remaining budget with amount and be positive" ); - let remaining = decrease_remaining_budget(&mut redis, campaign, amount) + let remaining = campaign_remaining + .decrease_by(campaign, amount) .await .expect("Should decrease campaign remaining"); assert_eq!( @@ -702,6 +673,7 @@ pub mod insert_events { async fn test_spending_for_events_with_enough_remaining_budget() { let mut redis = TESTS_POOL.get().await.expect("Should get redis connection"); let database = DATABASE_POOL.get().await.expect("Should get a DB pool"); + let campaign_remaining = CampaignRemaining::new(redis.connection.clone()); setup_test_migrations(database.pool.clone()) .await @@ -719,7 +691,7 @@ pub mod insert_events { { let spend_event = spend_for_event( &database.pool, - redis.connection.clone(), + &campaign_remaining, &campaign, publisher, leader, @@ -745,7 +717,7 @@ pub mod insert_events { let spend_event = spend_for_event( &database.pool, - redis.connection.clone(), + &campaign_remaining, &campaign, publisher, leader, @@ -765,8 +737,9 @@ pub mod insert_events { // Follower fee: 100 // Follower payout: 300 * 100 / 1000 = 30 assert_eq!( - 10_640_i64, - get_campaign_remaining(&mut redis.connection, campaign.id) + Some(10_640_i64), + campaign_remaining + .get_remaining_opt(campaign.id) .await .expect("Should have key") )