Skip to content

Implement async pagination #10021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Nov 21, 2024
Merged

Implement async pagination #10021

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
93cec6e
controllers/helpers/pagination: Implement `async_load()` fns
Turbo87 Nov 20, 2024
24da935
controllers/keyword: Remove `spawn_blocking()` call
Turbo87 Nov 20, 2024
86c143a
controllers/user/me: Remove `spawn_blocking()` call
Turbo87 Nov 20, 2024
afbc2dd
controllers/krate/search: Replace `OnceCell` with `OnceLock`
Turbo87 Nov 20, 2024
1302d73
controllers/krate/search: Remove `spawn_blocking()` call
Turbo87 Nov 20, 2024
1747c55
controllers/helpers/pagination: Remove sync `load()` fns
Turbo87 Nov 20, 2024
7d9347c
controllers/helpers/pagination: Remove obsolete `diesel::RunQueryDsl`…
Turbo87 Nov 21, 2024
fb99e47
controllers/helpers/pagination: Use regular `diesel` prelude
Turbo87 Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 45 additions & 20 deletions src/controllers/helpers/pagination.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@ use crate::models::helpers::with_count::*;
use crate::util::errors::{bad_request, AppResult};
use crate::util::{HeaderMapExt, RequestUtils};

use crate::util::diesel::prelude::*;
use base64::{engine::general_purpose, Engine};
use diesel::pg::Pg;
use diesel::prelude::*;
use diesel::query_builder::{AstPass, Query, QueryFragment, QueryId};
use diesel::query_dsl::LoadQuery;
use diesel::sql_types::BigInt;
use diesel_async::AsyncPgConnection;
use futures_util::future::BoxFuture;
use futures_util::{FutureExt, TryStreamExt};
use http::header;
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -250,16 +252,29 @@ pub(crate) struct PaginatedQuery<T> {
}

impl<T> PaginatedQuery<T> {
pub(crate) fn load<'a, U, Conn>(self, conn: &mut Conn) -> QueryResult<Paginated<U>>
pub fn load<'a, U>(
self,
conn: &'a mut AsyncPgConnection,
) -> BoxFuture<'a, QueryResult<Paginated<U>>>
where
Self: LoadQuery<'a, Conn, WithCount<U>>,
Self: diesel_async::methods::LoadQuery<'a, AsyncPgConnection, WithCount<U>>,
T: 'a,
U: Send + 'a,
{
Comment on lines +255 to 263
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I admit that I'm unsure about whether the lifetime 'a here remains exactly the same as 'conn and 'query in diesel-async.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no clue. I followed the suggestions of the compiler (when it wasn't crashing) and somehow made it work that way 😅

use diesel_async::methods::LoadQuery;

let options = self.options.clone();
let records_and_total = self.internal_load(conn)?.collect::<QueryResult<_>>()?;
Ok(Paginated {
records_and_total,
options,
})
let future = self.internal_load(conn);

async move {
let records_and_total = future.await?.try_collect().await?;

Ok(Paginated {
records_and_total,
options,
})
}
.boxed()
}
}

Expand All @@ -272,8 +287,6 @@ impl<T: Query> Query for PaginatedQuery<T> {
type SqlType = (T::SqlType, BigInt);
}

impl<T, DB> diesel::RunQueryDsl<DB> for PaginatedQuery<T> {}

impl<T> QueryFragment<Pg> for PaginatedQuery<T>
where
T: QueryFragment<Pg>,
Expand Down Expand Up @@ -366,8 +379,6 @@ impl<
type SqlType = (T::SqlType, BigInt);
}

impl<T, C, DB> diesel::RunQueryDsl<DB> for PaginatedQueryWithCountSubq<T, C> {}

impl<T, C> QueryFragment<Pg> for PaginatedQueryWithCountSubq<T, C>
where
T: QueryFragment<Pg>,
Expand All @@ -390,16 +401,30 @@ where
}

impl<T, C> PaginatedQueryWithCountSubq<T, C> {
pub(crate) fn load<'a, U, Conn>(self, conn: &mut Conn) -> QueryResult<Paginated<U>>
pub fn load<'a, U>(
self,
conn: &'a mut AsyncPgConnection,
) -> BoxFuture<'a, QueryResult<Paginated<U>>>
where
Self: LoadQuery<'a, Conn, WithCount<U>>,
Self: diesel_async::methods::LoadQuery<'a, AsyncPgConnection, WithCount<U>> + Send,
C: 'a,
T: 'a,
U: Send + 'a,
{
use diesel_async::methods::LoadQuery;

let options = self.options.clone();
let records_and_total = self.internal_load(conn)?.collect::<QueryResult<_>>()?;
Ok(Paginated {
records_and_total,
options,
})
let future = self.internal_load(conn);

async move {
let records_and_total = future.await?.try_collect().await?;

Ok(Paginated {
records_and_total,
options,
})
}
.boxed()
}
}

Expand Down
31 changes: 12 additions & 19 deletions src/controllers/keyword.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,12 @@ use crate::app::AppState;
use crate::controllers::helpers::pagination::PaginationOptions;
use crate::controllers::helpers::{pagination::Paginated, Paginate};
use crate::models::Keyword;
use crate::tasks::spawn_blocking;
use crate::util::errors::AppResult;
use crate::views::EncodableKeyword;
use axum::extract::{Path, Query};
use axum_extra::json;
use axum_extra::response::ErasedJson;
use diesel::prelude::*;
use diesel_async::async_connection_wrapper::AsyncConnectionWrapper;
use http::request::Parts;

#[derive(Deserialize)]
Expand All @@ -30,23 +28,18 @@ pub async fn index(state: AppState, qp: Query<IndexQuery>, req: Parts) -> AppRes

let query = query.pages_pagination(PaginationOptions::builder().gather(&req)?);

let conn = state.db_read().await?;
spawn_blocking(move || {
let conn: &mut AsyncConnectionWrapper<_> = &mut conn.into();

let data: Paginated<Keyword> = query.load(conn)?;
let total = data.total();
let kws = data
.into_iter()
.map(Keyword::into)
.collect::<Vec<EncodableKeyword>>();

Ok(json!({
"keywords": kws,
"meta": { "total": total },
}))
})
.await?
let mut conn = state.db_read().await?;
let data: Paginated<Keyword> = query.load(&mut conn).await?;
let total = data.total();
let kws = data
.into_iter()
.map(Keyword::into)
.collect::<Vec<EncodableKeyword>>();

Ok(json!({
"keywords": kws,
"meta": { "total": total },
}))
}

/// Handles the `GET /keywords/:keyword_id` route.
Expand Down
Loading