From b0b7b76e8f6bb208e343029b95519bf104936104 Mon Sep 17 00:00:00 2001 From: Zhenbo Li <3221521+Endle@users.noreply.github.com> Date: Fri, 6 Sep 2024 21:33:19 -0400 Subject: [PATCH] Append LLM result into DOM (#144) --- fireSeqSearch_addon/main.js | 165 ++++++++++-------- .../src/http_client/endpoints.rs | 11 +- fire_seq_search_server/src/local_llm/mod.rs | 55 +++--- fire_seq_search_server/src/main.rs | 3 +- .../src/query_engine/mod.rs | 10 ++ 5 files changed, 151 insertions(+), 93 deletions(-) diff --git a/fireSeqSearch_addon/main.js b/fireSeqSearch_addon/main.js index 2e413b2..93df9f7 100644 --- a/fireSeqSearch_addon/main.js +++ b/fireSeqSearch_addon/main.js @@ -142,72 +142,120 @@ function parseRawList(rawSearchResult) { return hits; } -function createTitleBarDom(count) { - const titleBar = createElementWithText("div"); - titleBar.classList.add('fireSeqSearchTitleBar'); - const hitCount = `We found ${count.toString()} results in your logseq notebook`; - titleBar.insertAdjacentHTML("afterbegin",hitCount); - - function setSummaryState(cl, state) { - let prop = 'none'; - if (state) { prop = ''; } - for (const el of document.querySelectorAll(cl)) { - el.style.display=prop; +async function processLlmSummary(serverInfo, parsedSearchResult, fireDom) { + + const doneListApi = "http://127.0.0.1:3030/llm_done_list"; + let list = await fetch(doneListApi); + list = await list.text(); + list = JSON.parse(list); + + const findByTitle = function(title) { + const ul = fireDom.querySelector( ".fireSeqSearchHitList" ); + if (ul === null) return null; + for (const child of ul.children) { + const liTitle = child.firstChild.text; + if (title === liTitle) { + return child; + } } - } - - let btn = document.createElement("button"); - btn.classList.add("hideSummary"); - let text = document.createTextNode("Hide Summary"); - btn.appendChild(text); - btn.onclick = function () { - setSummaryState(".fireSeqSearchHitSummary", false); - setSummaryState(".fireSeqSearchLlmSummary", false); - }; - titleBar.appendChild(btn); - - btn = document.createElement("button"); - btn.classList.add("showSummary"); - text = document.createTextNode("Summary"); - btn.appendChild(text); - btn.onclick = function () { - setSummaryState(".fireSeqSearchHitSummary", true); - setSummaryState(".fireSeqSearchLlmSummary", false); + return null; }; - titleBar.appendChild(btn); - + const setLlmResult = function (title, llmSummary) { + const targetRow = findByTitle(title); + if (targetRow === null) { + consoleLogForDebug("Error! Can't find dom for ", title); + return; + } + if (targetRow.querySelector( ".fireSeqSearchLlmSummary" ) != null) { + consoleLogForDebug("Skip. We have the summary for ", title); + return; + } - btn = document.createElement("button"); - btn.classList.add("showLlm"); - text = document.createTextNode("LLM"); - btn.appendChild(text); - btn.onclick = function () { - setSummaryState(".fireSeqSearchHitSummary", false); - setSummaryState(".fireSeqSearchLlmSummary", true); + const summary = createElementWithText("span", ""); + summary.innerHTML = llmSummary; + summary.classList.add('fireSeqSearchLlmSummary'); + targetRow.appendChild(summary); }; - titleBar.appendChild(btn); + for (const record of parsedSearchResult) { + const title = record.title; + if (!list.includes(title)) { + consoleLogForDebug("Not ready, skip" + title); + continue; + } + // TODO remove hard code port + const llm_api = "http://127.0.0.1:3030/summarize/" + title; + let sum = await fetch(llm_api); + sum = await sum.text(); + setLlmResult(title, sum); + } +} - return titleBar; -} -function createFireSeqDom(count) { +function createFireSeqDom(serverInfo, parsedSearchResult) { + const count = parsedSearchResult.length; const div = document.createElement("div"); div.setAttribute("id", fireSeqSearchDomId); - const bar = createTitleBarDom(count); + + const createTitleBarDom = function () { + const titleBar = createElementWithText("div"); + titleBar.classList.add('fireSeqSearchTitleBar'); + const hitCount = `We found ${count.toString()} results in your logseq notebook`; + titleBar.insertAdjacentHTML("afterbegin",hitCount); + + function setSummaryState(cl, state) { + let prop = 'none'; + if (state) { prop = ''; } + for (const el of document.querySelectorAll(cl)) { + el.style.display=prop; + } + } + let btn = document.createElement("button"); + btn.classList.add("hideSummary"); + let text = document.createTextNode("Hide Summary"); + btn.appendChild(text); + btn.onclick = function () { + setSummaryState(".fireSeqSearchHitSummary", false); + setSummaryState(".fireSeqSearchLlmSummary", false); + }; + titleBar.appendChild(btn); + + btn = document.createElement("button"); + btn.classList.add("showSummary"); + text = document.createTextNode("Summary"); + btn.appendChild(text); + btn.onclick = function () { + setSummaryState(".fireSeqSearchHitSummary", true); + setSummaryState(".fireSeqSearchLlmSummary", false); + }; + titleBar.appendChild(btn); + + btn = document.createElement("button"); + btn.classList.add("showLlm"); + text = document.createTextNode("LLM"); + btn.appendChild(text); + btn.onclick = function () { + setSummaryState(".fireSeqSearchHitSummary", false); + setSummaryState(".fireSeqSearchLlmSummary", true); + processLlmSummary(serverInfo, parsedSearchResult, div); + }; + titleBar.appendChild(btn); + return titleBar; + }; + const bar = createTitleBarDom(); div.appendChild(bar); return div; } async function appendResultToSearchResult(serverInfo, parsedSearchResult, dom) { const firefoxExtensionUserOption = await checkUserOptions(); - consoleLogForDebug('Loaded user option: ' + JSON.stringify(firefoxExtensionUserOption)); - function buildListItems(parsedSearchResult) { const hitList = document.createElement("ul"); + hitList.classList.add('fireSeqSearchHitList'); for (const record of parsedSearchResult) { const li = createElementWithText("li", ""); + li.classList.add('fireSeqSearchHitListItem'); if (firefoxExtensionUserOption.ShowScore) { const score = createElementWithText("span", String(record.score)); li.appendChild(score); @@ -252,18 +300,6 @@ async function appendResultToSearchResult(serverInfo, parsedSearchResult, dom) { insertDivToWebpage(dom); } -async function processLlmSummary(serverInfo, parsedSearchResult, dom) { - for (const record of parsedSearchResult) { - // TODO remove hard code port - const llm_api = "http://127.0.0.1:3030/summarize/" + record.title; - console.log("llm called"); - console.log(record.title); - const response = await fetch(llm_api); - const text = await response.text(); - console.log(text); - } -} - async function mainProcess(fetchResultArray) { consoleLogForDebug("main process"); @@ -272,18 +308,10 @@ async function mainProcess(fetchResultArray) { consoleLogForDebug(serverInfo); const parsedSearchResult = parseRawList(rawSearchResult); - console.log("in main"); - console.log(rawSearchResult); - console.log(parsedSearchResult); - - const fireDom = createFireSeqDom(parsedSearchResult.length); + const fireDom = createFireSeqDom(serverInfo, parsedSearchResult); appendResultToSearchResult(serverInfo, parsedSearchResult, fireDom); - if (serverInfo.llm_enabled) { - consoleLogForDebug("llm"); - processLlmSummary(serverInfo, parsedSearchResult, fireDom); - } } @@ -318,7 +346,6 @@ function getSearchParameterFromCurrentPage() { (function() { const searchParameter = getSearchParameterFromCurrentPage(); - addGlobalStyle(fireSeqSearchScriptCSS); //https://gomakethings.com/waiting-for-multiple-all-api-responses-to-complete-with-the-vanilla-js-promise.all-method/ @@ -328,9 +355,7 @@ function getSearchParameterFromCurrentPage() { ]).then(function (responses) { return Promise.all(responses.map(function (response) {return response.json();})); }).then(function (data) { - //consoleLogForDebug(data); mainProcess(data); - //return appendResultToSearchResult(data); }).then((_e) => { const highlightedItems = document.querySelectorAll('.fireSeqSearchHighlight'); consoleLogForDebug(highlightedItems); diff --git a/fire_seq_search_server/src/http_client/endpoints.rs b/fire_seq_search_server/src/http_client/endpoints.rs index 6d78417..58f0899 100644 --- a/fire_seq_search_server/src/http_client/endpoints.rs +++ b/fire_seq_search_server/src/http_client/endpoints.rs @@ -1,5 +1,5 @@ use std::sync::Arc; -use log::debug; +use log::{debug, info}; use crate::query_engine::{QueryEngine, ServerInformation}; use axum::Json; @@ -30,6 +30,15 @@ pub async fn summarize( Html(r.await) } +pub async fn get_llm_done_list( + State(engine_arc): State> + ) -> Html{ + + info!("get list endpoint called"); + let r = engine_arc.get_llm_done_list(); + Html(r.await) +} + pub async fn generate_word_cloud(State(engine_arc): State>) -> Html { let div_id = "fireSeqSearchWordcloudRawJson"; diff --git a/fire_seq_search_server/src/local_llm/mod.rs b/fire_seq_search_server/src/local_llm/mod.rs index 43f69a2..4a7c769 100644 --- a/fire_seq_search_server/src/local_llm/mod.rs +++ b/fire_seq_search_server/src/local_llm/mod.rs @@ -46,6 +46,13 @@ pub struct Usage { pub total_tokens: i64, } +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct HealthCheck { + pub slots_idle: i64, + pub slots_processing: i64, + pub status: String, +} + // End genereated const LLM_SERVER_PORT: &str = "8081"; // TODO Remove this magic number @@ -165,7 +172,6 @@ impl LlmEngine { impl LlmEngine{ pub async fn summarize(&self, full_text: &str) -> String { - info!("summarize called"); //http://localhost:8080/completion let ep = self.endpoint.to_owned() + "/v1/chat/completions"; let data = Self::build_data(full_text); @@ -175,17 +181,12 @@ impl LlmEngine{ .send() .await .unwrap(); - //info!(" response {:?}", &res); let content = res.text().await.unwrap(); - //info!(" text {:?}", &content); let parsed: LlamaResponse = serde_json::from_str(&content).unwrap(); - //info!(" parsed {:?}", &parsed); let v = parsed.choices; let v0 = v.into_iter().next().unwrap(); v0.message.content - - - //TODO remove unwrap + //TODO remove unwrap } pub async fn post_summarize_job(&self, doc: DocData) { @@ -196,13 +197,19 @@ impl LlmEngine{ } pub async fn call_llm_engine(&self) { + + let health = self.health().await.unwrap(); + if health.slots_idle == 0 { + info!("No valid slot, continue"); + return; + } + let mut next_job: Option = None; let mut jcache = self.job_cache.lock().await;//.unwrap(); next_job = jcache.job_queue.pop_front(); drop(jcache); - let doc = match next_job { Some(x) => x, None => { return; }, @@ -216,14 +223,14 @@ impl LlmEngine{ } drop(jcache); + info!("Start summarize job: {}", &title); let summarize_result = self.summarize(&doc.body).await; + info!("Finished summarize job: {}", &title); let mut jcache = self.job_cache.lock().await;//.unwrap(); next_job = jcache.job_queue.pop_front(); - info!("get summarize result {}", &title); jcache.done_job.insert(title, summarize_result); drop(jcache); - } pub async fn quick_fetch(&self, title: &str) -> Option { @@ -231,16 +238,24 @@ impl LlmEngine{ return jcache.done_job.get(title).cloned(); } - pub async fn health(&self) -> Result<(), Box> { - info!("Calling health check"); - let resp = reqwest::get(self.endpoint.to_owned() + "/health") - .await? - .headers().to_owned() - //.status() - //.text().await? - ; - info!("Health check: {:#?}", resp); - Ok(()) + pub async fn get_llm_done_list(&self) -> Vec { + let mut r = Vec::new(); + let jcache = self.job_cache.lock().await; + for (title, _text) in &jcache.done_job { + info!("already done : {}", &title); + r.push(title.to_owned()); + } + return r; + } + + pub async fn health(&self) -> Result> { + let res = self.client.get(self.endpoint.to_owned() + "/health") + .send() + .await + .unwrap(); + let content = res.text().await.unwrap(); + let parsed: HealthCheck = serde_json::from_str(&content).unwrap(); + Ok(parsed) } } diff --git a/fire_seq_search_server/src/main.rs b/fire_seq_search_server/src/main.rs index 800b644..a9c1ea4 100644 --- a/fire_seq_search_server/src/main.rs +++ b/fire_seq_search_server/src/main.rs @@ -78,12 +78,10 @@ async fn main() { engine.llm = Some(llm_arc); let poll_handle = tokio::spawn( async move { - info!("inside main loop"); loop { llm_poll.call_llm_engine().await; let wait_llm = tokio::time::Duration::from_millis(500); tokio::time::sleep(wait_llm).await; - info!("main loop: poll again"); } }); // poll_handle.await; @@ -97,6 +95,7 @@ async fn main() { .route("/server_info", get(endpoints::get_server_info)) .route("/wordcloud", get(endpoints::generate_word_cloud)) .route("/summarize/:title", get(endpoints::summarize)) + .route("/llm_done_list", get(endpoints::get_llm_done_list)) .with_state(engine_arc.clone()); let listener = tokio::net::TcpListener::bind(&engine_arc.server_info.host) diff --git a/fire_seq_search_server/src/query_engine/mod.rs b/fire_seq_search_server/src/query_engine/mod.rs index 62fe961..cb7c021 100644 --- a/fire_seq_search_server/src/query_engine/mod.rs +++ b/fire_seq_search_server/src/query_engine/mod.rs @@ -166,6 +166,16 @@ impl QueryEngine { "LLM turned off".to_owned() } } + pub async fn get_llm_done_list(&self) -> String { + if cfg!(feature="llm") { + let llm = self.llm.as_ref().unwrap(); + let result = &llm.get_llm_done_list().await; + let json = serde_json::to_string(&result).unwrap(); + return json; + } else { + "LLM turned off".to_owned() + } + } } fn term_preprocess(term:String) -> String {