diff --git a/fire_seq_search_server/obsidian.sh b/fire_seq_search_server/obsidian.sh index 6baf5cd..22eab6a 100755 --- a/fire_seq_search_server/obsidian.sh +++ b/fire_seq_search_server/obsidian.sh @@ -1,5 +1,5 @@ set -e -cargo build +cargo build --features llm rm ./fire_seq_search_server -f cp --force target/debug/fire_seq_search_server ./fire_seq_search_server diff --git a/fire_seq_search_server/src/lib.rs b/fire_seq_search_server/src/lib.rs index 7102a94..6752d9c 100644 --- a/fire_seq_search_server/src/lib.rs +++ b/fire_seq_search_server/src/lib.rs @@ -10,6 +10,7 @@ pub mod local_llm; use log::{debug, info}; use crate::query_engine::ServerInformation; +use crate::query_engine::NotebookSoftware::Logseq; #[macro_use] @@ -168,7 +169,7 @@ pub fn generate_server_info_for_test() -> ServerInformation { show_summary_single_line_chars_limit: 0, parse_pdf_links: false, exclude_zotero_items: false, - obsidian_md: false, + software: Logseq, convert_underline_hierarchy: true, host: "127.0.0.1:22024".to_string(), llm_enabled: false, diff --git a/fire_seq_search_server/src/load_notes/mod.rs b/fire_seq_search_server/src/load_notes/mod.rs index 96d295e..1de95a3 100644 --- a/fire_seq_search_server/src/load_notes/mod.rs +++ b/fire_seq_search_server/src/load_notes/mod.rs @@ -86,62 +86,6 @@ fn list_directory(path: Cow<'_, str>, recursive: bool) -> Vec { return result; } -/* -pub fn read_all_notes(server_info: &ServerInformation) -> Vec<(String, String)> { - // I should remove the unwrap and convert it into map - let path: &str = &server_info.notebook_path; - let path = path.to_owned(); - let pages_path = if server_info.obsidian_md { - path.clone() - } else{ - path.clone() + "/pages" - }; - - - let mut pages: Vec<(String, String)> = Vec:: new(); - - let pages_tmp: Vec<(String, String)> = read_specific_directory(&pages_path).par_iter() - .map(|(title,md)| { - let content = crate::markdown_parser::parse_logseq_notebook(md, title, server_info); - (title.to_string(), content) - }).collect(); //silly collect. - - if server_info.exclude_zotero_items { - error!("exclude zotero disabled"); - } - /* - for (file_name, contents) in pages_tmp { - // info!("File Name: {}", &file_name); - if server_info.exclude_zotero_items && file_name.starts_with('@') { - continue; - } - pages.push((file_name,contents)); - } - */ - if server_info.enable_journal_query { - info!("Loading journals"); - let journals_page = path.clone() + "/journals"; - let journals:Vec<(String, String)> - = read_specific_directory(&journals_page).par_iter() - .map(|(title,md)| { - let content = crate::markdown_parser::parse_logseq_notebook(md, title, server_info); - let tantivy_title = JOURNAL_PREFIX.to_owned() + &title; - (tantivy_title, content) - }).collect(); //silly collect. - - - for (file_name, contents) in journals { - pages.push((file_name,contents)); - } - - } - - pages - -} - - -*/ diff --git a/fire_seq_search_server/src/main.rs b/fire_seq_search_server/src/main.rs index b5f39ec..37f7951 100644 --- a/fire_seq_search_server/src/main.rs +++ b/fire_seq_search_server/src/main.rs @@ -2,6 +2,7 @@ use log::info; use fire_seq_search_server::query_engine::{QueryEngine, ServerInformation}; use fire_seq_search_server::local_llm::LlmEngine; +use fire_seq_search_server::query_engine::NotebookSoftware::*; use clap::Parser; @@ -118,6 +119,10 @@ fn build_server_info(args: Cli) -> ServerInformation { } }; let host: String = args.host.clone().unwrap_or_else(|| "127.0.0.1:3030".to_string()); + let mut software = Logseq; + if args.obsidian_md { + software = Obsidian; + } ServerInformation{ notebook_path: args.notebook_path, notebook_name, @@ -127,7 +132,7 @@ fn build_server_info(args: Cli) -> ServerInformation { args.show_summary_single_line_chars_limit, parse_pdf_links: args.parse_pdf_links, exclude_zotero_items:args.exclude_zotero_items, - obsidian_md: args.obsidian_md, + software, convert_underline_hierarchy: true, host, llm_enabled: cfg!(feature="llm"), diff --git a/fire_seq_search_server/src/post_query/app_uri.rs b/fire_seq_search_server/src/post_query/app_uri.rs index 7859125..c5b8d3f 100644 --- a/fire_seq_search_server/src/post_query/app_uri.rs +++ b/fire_seq_search_server/src/post_query/app_uri.rs @@ -3,9 +3,11 @@ use crate::post_query::logseq_uri::generate_logseq_uri; use crate::post_query::obsidian_uri::generate_obsidian_uri; use crate::query_engine::ServerInformation; +use crate::query_engine::NotebookSoftware::Obsidian; + // Maybe I should wrap them with the same interface? -Zhenbo Li 2023-Feb-05 pub fn generate_uri(title: &str, is_page_hit: &bool, server_info: &ServerInformation) -> String { - if server_info.obsidian_md { + if server_info.software == Obsidian { info!("Generating Obsidian URI for {}", title); if !is_page_hit { error!("Journal is unsupported for Obsidian yet"); @@ -16,4 +18,4 @@ pub fn generate_uri(title: &str, is_page_hit: &bool, server_info: &ServerInforma return generate_logseq_uri(&title, &is_page_hit, &server_info); -} \ No newline at end of file +} diff --git a/fire_seq_search_server/src/query_engine/mod.rs b/fire_seq_search_server/src/query_engine/mod.rs index 51a297e..8d5655c 100644 --- a/fire_seq_search_server/src/query_engine/mod.rs +++ b/fire_seq_search_server/src/query_engine/mod.rs @@ -9,6 +9,12 @@ use std::sync::Arc; use std::borrow::Cow; +#[derive(Debug, Clone, serde::Serialize,PartialEq)] +pub enum NotebookSoftware { + Logseq, + Obsidian, +} + // This struct should be immutable when the program starts running #[derive(Debug, Clone, serde::Serialize)] pub struct ServerInformation { @@ -19,7 +25,7 @@ pub struct ServerInformation { pub show_summary_single_line_chars_limit: usize, pub parse_pdf_links: bool, pub exclude_zotero_items:bool, - pub obsidian_md: bool, + pub software: NotebookSoftware, /// Experimental. Not sure if I should use this global config - 2022-12-30 pub convert_underline_hierarchy: bool, @@ -278,41 +284,6 @@ fn build_reader_parser(index: &tantivy::Index, document_setting: &DocumentSettin (reader, query_parser) } -/* -fn indexing_documents(server_info: &ServerInformation, - document_setting: &DocumentSetting, - pages:&Vec) -> tantivy::Index { - - let schema = &document_setting.schema; - let index = tantivy::Index::create_in_ram(schema.clone()); - - index.tokenizers().register(TOKENIZER_ID, document_setting.tokenizer.clone()); - - let mut index_writer = index.writer(50_000_000).unwrap(); - - - if server_info.obsidian_md { - warn!("Obsidian mode."); - assert!(!server_info.enable_journal_query); - } - - let title = schema.get_field("title").unwrap(); - let body = schema.get_field("body").unwrap(); - - - for article in pages { - index_writer.add_document( - tantivy::doc!{ title => article.file_name.clone(), - body => article.content.clone()} - ).unwrap(); - } - index_writer.commit().unwrap(); - index -} -*/ - - - fn build_document_setting() -> DocumentSetting { let (schema, tokenizer) = build_schema_tokenizer(); DocumentSetting{