From 5e8163f1faaa03dc9d21ddf7963cce62af766437 Mon Sep 17 00:00:00 2001 From: Joseph Livesey Date: Wed, 3 Jan 2024 13:47:32 -0500 Subject: [PATCH] feat: replace JSON store with support for importing from files --- crates/api/src/lib.rs | 6 +- crates/common/src/commands.rs | 2 +- crates/gust/src/cli.rs | 10 +- crates/gust/src/command.rs | 6 +- .../src/{json => import_store}/mod.rs | 180 ++++-------------- crates/persistence/src/lib.rs | 2 +- .../src/{json/migrate.rs => sqlite/import.rs} | 5 +- crates/persistence/src/sqlite/migrations.rs | 16 ++ crates/persistence/src/sqlite/mod.rs | 40 ++-- crates/persistence/src/store.rs | 38 +--- 10 files changed, 105 insertions(+), 200 deletions(-) rename crates/persistence/src/{json => import_store}/mod.rs (80%) rename crates/persistence/src/{json/migrate.rs => sqlite/import.rs} (97%) create mode 100644 crates/persistence/src/sqlite/migrations.rs diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 6d2301e..325ebb6 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -120,7 +120,7 @@ pub enum ApiResponse { FetchedRecipe((Recipe, Ingredients)), ItemAlreadyAdded(Name), Items(Items), - JsonToSqlite, + ImportToSqlite, List(List), NothingReturned(ApiCommand), Recipes(Vec), @@ -164,7 +164,7 @@ impl Display for ApiResponse { } Ok(()) } - Self::JsonToSqlite => writeln!(f, "\nJSON to SQLite data store migration successful"), + Self::ImportToSqlite => writeln!(f, "\nJSON to SQLite data store migration successful"), Self::List(list) => { writeln!(f)?; for item in list.items() { @@ -216,7 +216,7 @@ impl From for ApiResponse { StoreResponse::FetchedRecipe(item) => Self::FetchedRecipe(item), StoreResponse::ItemAlreadyAdded(item) => Self::ItemAlreadyAdded(item), StoreResponse::Items(item) => Self::Items(item), - StoreResponse::JsonToSqlite => Self::JsonToSqlite, + StoreResponse::ImportToSqlite => Self::ImportToSqlite, StoreResponse::List(item) => Self::List(item), StoreResponse::NothingReturned(item) => Self::NothingReturned(item), StoreResponse::Recipes(item) => Self::Recipes(item), diff --git a/crates/common/src/commands.rs b/crates/common/src/commands.rs index 6eb1a2a..1b818ce 100644 --- a/crates/common/src/commands.rs +++ b/crates/common/src/commands.rs @@ -10,7 +10,7 @@ pub enum ApiCommand { Add(Add), Delete(Delete), FetchRecipe(Url), - MigrateJsonDbToSqlite, + ImportFromJson, Read(Read), Update(Update), } diff --git a/crates/gust/src/cli.rs b/crates/gust/src/cli.rs index a511c01..fe73f37 100644 --- a/crates/gust/src/cli.rs +++ b/crates/gust/src/cli.rs @@ -188,10 +188,10 @@ fn update() -> Command { .subcommand(list().subcommand(refresh_list())) } -fn migrate() -> Command { - Command::new("migrate-json-store") +fn import() -> Command { + Command::new("import") .subcommand_required(false) - .about("migrate JSON store to Sqlite database") + .about("import from 'items.json' and 'list.json' files") } pub fn cli() -> Command { @@ -204,12 +204,12 @@ pub fn cli() -> Command { .subcommand(fetch()) .subcommand(read()) .subcommand(update()) - .subcommand(migrate()) + .subcommand(import()) .arg( Arg::new("store") .long("database") .num_args(1) - .value_parser(["json", "sqlite", "sqlite-inmem"]) + .value_parser(["sqlite", "sqlite-inmem"]) .default_value("sqlite") .help("which database to use"), ) diff --git a/crates/gust/src/command.rs b/crates/gust/src/command.rs index 8b84d69..d56fbd8 100644 --- a/crates/gust/src/command.rs +++ b/crates/gust/src/command.rs @@ -13,7 +13,7 @@ pub enum GustCommand { Add(Add), Delete(Delete), FetchRecipe(Url), - MigrateJsonDbToSqlite, + ImportFromJson, Read(Read), Update(Update), } @@ -117,7 +117,7 @@ impl TryFrom for GustCommand { } _ => unimplemented!(), })), - Some(("migrate-json-store", _)) => Ok(GustCommand::MigrateJsonDbToSqlite), + Some(("import", _)) => Ok(GustCommand::ImportFromJson), _ => unreachable!(), } } @@ -129,7 +129,7 @@ impl From for ApiCommand { GustCommand::Add(cmd) => Self::Add(cmd), GustCommand::Delete(cmd) => Self::Delete(cmd), GustCommand::FetchRecipe(cmd) => Self::FetchRecipe(cmd), - GustCommand::MigrateJsonDbToSqlite => Self::MigrateJsonDbToSqlite, + GustCommand::ImportFromJson => Self::ImportFromJson, GustCommand::Read(cmd) => Self::Read(cmd), GustCommand::Update(cmd) => Self::Update(cmd), } diff --git a/crates/persistence/src/json/mod.rs b/crates/persistence/src/import_store/mod.rs similarity index 80% rename from crates/persistence/src/json/mod.rs rename to crates/persistence/src/import_store/mod.rs index 7c2549b..972f1d3 100644 --- a/crates/persistence/src/json/mod.rs +++ b/crates/persistence/src/import_store/mod.rs @@ -1,33 +1,23 @@ -pub mod migrate; - use std::{ collections::HashSet, fs::{self}, path::{Path, PathBuf}, }; -use common::{ - input::item_matches, - item::{Item, Name}, - items::Items, - list::List, - recipes::{Ingredients, Recipe}, - Load, -}; -use question::Answer; +use common::{items::Items, list::List, recipes::Recipe, Load}; -use crate::store::{Storage, StoreError, StoreResponse}; +use crate::store::StoreError; -pub const ITEMS_JSON_PATH: &str = "store.json"; +pub const ITEMS_JSON_PATH: &str = "items.json"; pub const LIST_JSON_PATH: &str = "list.json"; #[derive(Clone)] -pub struct JsonStore { +pub struct ImportStore { items: PathBuf, list: PathBuf, } -impl Default for JsonStore { +impl Default for ImportStore { fn default() -> Self { Self { items: PathBuf::from(ITEMS_JSON_PATH), @@ -36,7 +26,7 @@ impl Default for JsonStore { } } -impl JsonStore { +impl ImportStore { pub fn new() -> Self { Self::default() } @@ -51,122 +41,48 @@ impl JsonStore { self } - pub fn save_items(&self, object: impl serde::Serialize) -> Result<(), StoreError> { - let s = serde_json::to_string(&object)?; - - Ok(fs::write(&self.items, s)?) + pub fn items(&self) -> Result { + Ok(Items::from_json(&self.items)?) } - // TODO: I don't think it makes much sense to have these saved as separate JSON files. - pub fn save_list(&self, object: impl serde::Serialize) -> Result<(), StoreError> { - let s = serde_json::to_string(&object)?; - Ok(fs::write(&self.list, s)?) - } -} + pub fn recipes(&self) -> Result, StoreError> { + let mut recipes: HashSet = HashSet::new(); -impl Storage for JsonStore { - async fn add_item(&self, item: &Name) -> Result { - let mut groceries = Items::from_json(&self.items)?; + let groceries = Items::from_json(&self.items)?; - if groceries - .get_item_matches(item.as_str()) - .any(|item| matches!(item_matches(item), Answer::YES)) - { - eprintln!("Item already in library"); - Ok(StoreResponse::ItemAlreadyAdded(item.clone())) - } else { - let new_item = Item::new(item.as_str()); - groceries.add_item(new_item); - Ok(StoreResponse::AddedItem(item.clone())) + for item in groceries.collection() { + if let Some(item_recipes) = item.recipes() { + for recipe in item_recipes.iter().cloned() { + recipes.insert(recipe); + } + } } - } - - async fn add_checklist_item(&self, _item: &Name) -> Result { - todo!() - } - - async fn add_list_item(&self, _item: &Name) -> Result { - todo!() - } - - async fn add_list_recipe(&self, _recipe: &Recipe) -> Result { - todo!() - } - - async fn add_recipe( - &self, - _recipe: &Recipe, - _ingredients: &Ingredients, - ) -> Result { - todo!() - } - - async fn checklist(&self) -> Result { - todo!() - } - - async fn delete_checklist_item(&self, _item: &Name) -> Result { - todo!() - } - async fn delete_recipe(&self, _recipe: &Recipe) -> Result { - todo!() - } + for recipe in groceries.recipes().cloned() { + recipes.insert(recipe); + } - async fn items(&self) -> Result { - Ok(StoreResponse::Items(Items::from_json(&self.items)?)) - } + let list = List::from_json(&self.list)?; - async fn list(&self) -> Result { - Ok(StoreResponse::List(List::from_json(&self.list)?)) - } + for recipe in list.recipes().cloned() { + recipes.insert(recipe); + } - async fn refresh_list(&self) -> Result { - todo!() + Ok(recipes.into_iter().collect()) } - async fn recipe_ingredients(&self, recipe: &Recipe) -> Result { - let items = Items::from_json(&self.items)?; - let ingredients: Ingredients = items - .recipe_ingredients(recipe) - .map(|item| item.name()) - .cloned() - .collect(); - - Ok(StoreResponse::RecipeIngredients(Some(ingredients))) + pub fn list(&self) -> Result { + Ok(List::from_json(&self.list)?) } - async fn sections(&self) -> Result { - todo!() + pub fn save_items(&self, object: impl serde::Serialize) -> Result<(), StoreError> { + let s = serde_json::to_string(&object)?; + Ok(fs::write(&self.items, s)?) } - async fn recipes(&self) -> Result { - let mut recipes: HashSet = HashSet::new(); - - { - let groceries = Items::from_json(&self.items)?; - - for item in groceries.collection() { - if let Some(item_recipes) = item.recipes() { - for recipe in item_recipes.iter().cloned() { - recipes.insert(recipe); - } - } - } - - for recipe in groceries.recipes().cloned() { - recipes.insert(recipe); - } - } - - { - let list = List::from_json(&self.list)?; - - for recipe in list.recipes().cloned() { - recipes.insert(recipe); - } - } - Ok(StoreResponse::Recipes(recipes.into_iter().collect())) + pub fn save_list(&self, object: impl serde::Serialize) -> Result<(), StoreError> { + let s = serde_json::to_string(&object)?; + Ok(fs::write(&self.list, s)?) } } @@ -175,6 +91,7 @@ pub mod test { use super::*; use assert_fs::prelude::*; + use common::{item::Item, recipes::Recipe}; fn test_json_file() -> Result> { let file = assert_fs::NamedTempFile::new("test1.json")?; @@ -209,11 +126,8 @@ pub mod test { async fn items() -> Items { let file = test_json_file().unwrap(); - let store = JsonStore::new().with_items_path(file.path()); - let StoreResponse::Items(items) = store.items().await.unwrap() else { - todo!() - }; - items + let store = ImportStore::new().with_items_path(file.path()); + store.items().unwrap() } #[test] @@ -231,7 +145,7 @@ pub mod test { #[tokio::test] async fn test_save_items() -> Result<(), Box> { - let store = JsonStore::new().with_items_path(&PathBuf::from("test_groceries.json")); + let store = ImportStore::new().with_items_path(&PathBuf::from("test_groceries.json")); let items = Items::default(); insta::assert_json_snapshot!(items, @r#" { @@ -241,18 +155,13 @@ pub mod test { } "#); store.save_items(items)?; - match store.items().await.unwrap() { - StoreResponse::Items(items) => { - insta::assert_json_snapshot!(items, @r#" + insta::assert_json_snapshot!(store.items().unwrap(), @r#" { "sections": [], "collection": [], "recipes": [] } "#); - } - _ => panic!(), - } std::fs::remove_file(store.items)?; Ok(()) } @@ -480,11 +389,9 @@ pub mod test { #[tokio::test] async fn test_delete_item_from_list() -> Result<(), Box> { let file = create_test_checklist_json_file().unwrap(); - let store = JsonStore::new().with_list_path(file.path()); + let store = ImportStore::new().with_list_path(file.path()); - let StoreResponse::List(mut shopping_list) = store.list().await.unwrap() else { - todo!() - }; + let mut shopping_list = store.list().unwrap(); let item = Item::new("kumquats").with_section("fresh"); @@ -603,11 +510,8 @@ pub mod test { async fn checklist() -> List { let file = create_test_checklist_json_file().unwrap(); - let store = JsonStore::new().with_list_path(file.path()); - let StoreResponse::List(list) = store.list().await.unwrap() else { - todo!() - }; - list + let store = ImportStore::new().with_list_path(file.path()); + store.list().unwrap() } #[tokio::test] diff --git a/crates/persistence/src/lib.rs b/crates/persistence/src/lib.rs index a582af7..1066268 100644 --- a/crates/persistence/src/lib.rs +++ b/crates/persistence/src/lib.rs @@ -1,4 +1,4 @@ -pub mod json; +pub mod import_store; pub mod models; pub mod schema; pub mod sqlite; diff --git a/crates/persistence/src/json/migrate.rs b/crates/persistence/src/sqlite/import.rs similarity index 97% rename from crates/persistence/src/json/migrate.rs rename to crates/persistence/src/sqlite/import.rs index 8b66055..5268abc 100644 --- a/crates/persistence/src/json/migrate.rs +++ b/crates/persistence/src/sqlite/import.rs @@ -44,7 +44,10 @@ pub fn migrate_recipes( Ok(()) } -pub fn groceries(connection: &mut SqliteConnection, groceries: Items) -> Result<(), StoreError> { +pub fn migrate_items( + connection: &mut SqliteConnection, + groceries: Items, +) -> Result<(), StoreError> { let items_table = schema::items::table; let recipes_table = schema::recipes::table; let sections_table = schema::sections::table; diff --git a/crates/persistence/src/sqlite/migrations.rs b/crates/persistence/src/sqlite/migrations.rs new file mode 100644 index 0000000..41e3b83 --- /dev/null +++ b/crates/persistence/src/sqlite/migrations.rs @@ -0,0 +1,16 @@ +use diesel::sqlite::Sqlite; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; + +use crate::store::StoreError; + +pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations"); + +pub fn run_migrations(connection: &mut impl MigrationHarness) -> Result<(), StoreError> { + // This will run the necessary migrations. + // + // See the documentation for `MigrationHarness` for + // all available methods. + connection.run_pending_migrations(MIGRATIONS)?; + + Ok(()) +} diff --git a/crates/persistence/src/sqlite/mod.rs b/crates/persistence/src/sqlite/mod.rs index b4d4535..dfa047e 100644 --- a/crates/persistence/src/sqlite/mod.rs +++ b/crates/persistence/src/sqlite/mod.rs @@ -1,3 +1,6 @@ +mod import; +mod migrations; + use std::{env, ops::Deref}; use common::{ @@ -5,11 +8,11 @@ use common::{ list::List, recipes::{Ingredients, Recipe}, }; -use diesel::{prelude::*, r2d2::ConnectionManager, sqlite::Sqlite, SqliteConnection}; -use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use diesel::{prelude::*, r2d2::ConnectionManager, SqliteConnection}; use r2d2::{Pool, PooledConnection}; use crate::{ + import_store::ImportStore, models::{ self, Item, ItemInfo, NewChecklistItem, NewItem, NewItemRecipe, NewListItem, NewListRecipe, NewRecipe, RecipeModel, Section, @@ -18,6 +21,11 @@ use crate::{ store::{Storage, StoreError, StoreResponse}, }; +use self::{ + import::{migrate_items, migrate_recipes, migrate_sections}, + migrations::run_migrations, +}; + pub struct DbUri(String); impl Default for DbUri { @@ -82,18 +90,6 @@ impl Connection for DatabaseConnector { } } -pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations"); - -fn run_migrations(connection: &mut impl MigrationHarness) -> Result<(), StoreError> { - // This will run the necessary migrations. - // - // See the documentation for `MigrationHarness` for - // all available methods. - connection.run_pending_migrations(MIGRATIONS)?; - - Ok(()) -} - #[derive(Clone)] pub struct SqliteStore { pool: ConnectionPool, @@ -440,6 +436,22 @@ impl Storage for SqliteStore { .await? } + async fn import_from_json(&self) -> Result { + let import_store = ImportStore::default(); + let mut connection = self.connection()?; + let items = import_store.items()?; + let recipes = import_store.recipes()?; + tokio::task::spawn_blocking(move || { + connection.immediate_transaction(|connection| { + migrate_sections(connection)?; + migrate_recipes(connection, recipes)?; + migrate_items(connection, items)?; + Ok(StoreResponse::ImportToSqlite) + }) + }) + .await? + } + async fn items(&self) -> Result { use schema::items::dsl::items; diff --git a/crates/persistence/src/store.rs b/crates/persistence/src/store.rs index 518e703..9c29e28 100644 --- a/crates/persistence/src/store.rs +++ b/crates/persistence/src/store.rs @@ -18,13 +18,7 @@ use url::Url; use std::{error::Error, fmt::Debug, fmt::Display, str::FromStr}; -use crate::{ - json::{ - migrate::{groceries, migrate_recipes, migrate_sections}, - JsonStore, - }, - sqlite::{DbUri, SqliteStore}, -}; +use crate::sqlite::{DbUri, SqliteStore}; #[derive(Error, Debug)] pub enum StoreError { @@ -67,7 +61,6 @@ pub enum StoreError { #[derive(Debug)] pub enum StoreType { - Json, Sqlite, SqliteInMem, } @@ -75,7 +68,6 @@ pub enum StoreType { impl Display for StoreType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - StoreType::Json => write!(f, "json"), StoreType::Sqlite => write!(f, "sqlite"), StoreType::SqliteInMem => write!(f, "sqlite-inmem"), } @@ -87,7 +79,6 @@ impl FromStr for StoreType { fn from_str(s: &str) -> Result { match s { - "json" => Ok(Self::Json), "sqlite" => Ok(Self::Sqlite), "sqlite-inmem" => Ok(Self::SqliteInMem), _ => Err(StoreError::ParseStoreType( @@ -100,7 +91,6 @@ impl FromStr for StoreType { #[derive(Clone)] pub enum Store { - Json(JsonStore), Sqlite(SqliteStore), } @@ -108,7 +98,6 @@ impl Store { pub async fn from_store_type(store_type: StoreType) -> Result { use StoreType::*; match store_type { - Json => Ok(Self::Json(JsonStore::default())), Sqlite => Ok(Self::Sqlite(SqliteStore::new(DbUri::new()).await?)), SqliteInMem => Ok(Self::Sqlite(SqliteStore::new(DbUri::inmem()).await?)), } @@ -149,7 +138,6 @@ impl Store { async fn execute_transaction(&self, command: ApiCommand) -> Result { match self { - Self::Json(store) => store.execute_transaction(command).await, Self::Sqlite(store) => store.execute_transaction(command).await, } } @@ -196,9 +184,9 @@ pub enum StoreResponse { DeletedRecipe(Recipe), DeletedChecklistItem(Name), FetchedRecipe((Recipe, Ingredients)), + ImportToSqlite, ItemAlreadyAdded(Name), Items(Items), - JsonToSqlite, List(List), NothingReturned(ApiCommand), Recipes(Vec), @@ -213,7 +201,7 @@ pub(crate) trait Storage: Send + Sync + 'static { ApiCommand::Add(cmd) => self.add(cmd).await, ApiCommand::Delete(cmd) => self.delete(cmd).await, ApiCommand::FetchRecipe(url) => self.fetch_recipe(url).await, - ApiCommand::MigrateJsonDbToSqlite => self.migrate_json_store_to_sqlite().await, + ApiCommand::ImportFromJson => self.import_from_json().await, ApiCommand::Read(cmd) => self.read(cmd).await, ApiCommand::Update(cmd) => self.update(cmd).await, } @@ -272,25 +260,7 @@ pub(crate) trait Storage: Send + Sync + 'static { Ok(StoreResponse::FetchedRecipe((recipe, ingredients))) } - async fn migrate_json_store_to_sqlite(&self) -> Result { - let sqlite_store = SqliteStore::new(DbUri::new()).await?; - let mut connection = sqlite_store.connection()?; - let StoreResponse::Items(grocery_items) = self.items().await? else { - todo!() - }; - let StoreResponse::Recipes(recipes) = self.recipes().await? else { - todo!() - }; - tokio::task::spawn_blocking(move || { - connection.immediate_transaction(|connection| { - migrate_sections(connection)?; - migrate_recipes(connection, recipes)?; - groceries(connection, grocery_items)?; - Ok(StoreResponse::JsonToSqlite) - }) - }) - .await? - } + async fn import_from_json(&self) -> Result; // Create async fn add_item(&self, item: &Name) -> Result;