content extracted from json

fire_seq_search_server/Cargo.toml

@@ -66,3 +66,4 @@ pdf-extract-temporary-mitigation-panic = "0.7.1"
 sha256 = "1.5.0"
 reqwest = { version = "0.12", features = ["json"] }
 futures = "0.3"
+serde_derive = "1.0.209"

fire_seq_search_server/src/local_llm/mod.rs

@@ -5,6 +5,48 @@ use std::collections::HashMap;
 use std::collections::VecDeque;
 
 
+// Generated by https://transform.tools/json-to-rust-serde
+use serde_derive::Deserialize;
+use serde_derive::Serialize;
+use serde;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct OpenAiData {
+    pub model: String,
+    pub messages: Vec<Message>,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct LlamaResponse {
+    pub choices: Vec<Choice>,
+    pub created: i64,
+    pub id: String,
+    pub model: String,
+    pub object: String,
+    pub usage: Usage,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Choice {
+    pub finish_reason: String,
+    pub index: i64,
+    pub message: Message,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Message {
+    pub content: String,
+    pub role: String,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Usage {
+    pub completion_tokens: i64,
+    pub prompt_tokens: i64,
+    pub total_tokens: i64,
+}
+
+// End genereated
 
 const LLM_SERVER_PORT: &str = "8081"; // TODO Remove this magic number
 use std::sync::Arc;
@@ -39,19 +81,8 @@ pub struct LlmEngine {
     //job_cache :Arc<Mutex<HashMap<String, Option<String> >>>,
 }
 
-use serde::{Serialize, Deserialize};
 
-#[derive(Debug, Serialize, Deserialize)]
-pub struct OpenAiData {
-    pub model: String,
-    pub messages: Vec<Message>,
-}
 
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Message {
-    pub role: String,
-    pub content: String,
-}
 
 use tokio::task::yield_now;
 use tokio::task;
@@ -132,7 +163,6 @@ impl LlmEngine {
     }
 }
 
-    use axum::debug_handler;
 impl LlmEngine{
     pub async fn summarize(&self, full_text: &str) -> String {
         info!("summarize called");
@@ -145,10 +175,16 @@ impl LlmEngine{
             .send()
             .await
             .unwrap();
-        info!(" response {:?}", &res);
+        //info!(" response {:?}", &res);
         let content = res.text().await.unwrap();
-        info!(" text {:?}", &content);
-        content
+        //info!(" text {:?}", &content);
+        let parsed: LlamaResponse = serde_json::from_str(&content).unwrap();
+        //info!(" parsed {:?}", &parsed);
+        let v = parsed.choices;
+        let v0 = v.into_iter().next().unwrap();
+        v0.message.content
+
+
             //TODO remove unwrap
     }
 
@@ -157,10 +193,6 @@ impl LlmEngine{
         let mut jcache = self.job_cache.lock().await;//.unwrap();
         jcache.add(doc);
         drop(jcache);
-
-        //TODO why can't I call self.poll in this function?
-        //yield_now().await;
-        //self.poll().await;
     }
 
     pub async fn call_llm_engine(&self) {