Remove localServer /api/edge endpoint since the web app will directly…

… contact the llama server
ohmplatform · Jun 4, 2024 · 6daebe8 · 6daebe8
1 parent 87ab375
commit 6daebe8
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 93 deletions.
diff --git a/renderer/components/Chat/Chat.tsx b/renderer/components/Chat/Chat.tsx
@@ -184,21 +184,8 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
           })
           .join("\n");
 
-        const requestData = {
-          data: {
-            model: selectedModel.id,
-            prompt: messagesToSendString,
-            stream: true,
-            n_predict: 512,
-          },
-          endpoint: "completion",
-        };
-
-        let body = JSON.stringify(requestData);
-
         const controller = new AbortController();
-
-        const url = `http://localhost:${LOCAL_SERVER_PORT}/api/edge`;
+        const url = `http://127.0.0.1:8887/completion`;
         const response = await fetch(
           url,
 
@@ -208,7 +195,11 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
               "Content-Type": "application/json",
             },
             signal: controller.signal,
-            body,
+            body: JSON.stringify({
+              prompt: messagesToSendString,
+              stream: true,
+              n_predict: 512,
+            }),
           }
         );
 
@@ -245,7 +236,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
         const decoder = new TextDecoder();
         let done = false;
         let isFirst = true;
-        let text = "";
+        let text = '';
         while (!done) {
           if (stopConversationRef.current === true) {
             controller.abort();
@@ -255,35 +246,44 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
           const { value, done: doneReading } = await reader.read();
           done = doneReading;
           const chunkValue = decoder.decode(value);
-          text += chunkValue;
+
+          try {
+            if (chunkValue.startsWith('data: ')) {
+              const parsedChunkValue = JSON.parse(chunkValue.substring(6));
+              text += parsedChunkValue.content;
+            }
+          } catch (error) {
+            console.error("Parsing error:", error);
+          }
+
           if (isFirst) {
             isFirst = false;
             let updatedMessages: Message[] = [];
 
-            if (message.content === "continue") {
+            if (message.content === 'continue') {
               let lastMessage =
                 updatedConversation.messages[
                   updatedConversation.messages.length - 1
                 ];
 
               let updatedLastMessage = {
                 ...lastMessage,
-                content: lastMessage.content + "\n" + chunkValue,
+                content: lastMessage.content + '\n' + text,
               };
 
               updatedMessages = [
                 ...updatedConversation.messages.slice(
                   0,
-                  updatedConversation.messages.length - 1
+                  updatedConversation.messages.length - 1,
                 ),
                 updatedLastMessage,
               ];
             } else {
               updatedMessages = [
                 ...updatedConversation.messages,
                 {
-                  role: "assistant",
-                  content: chunkValue,
+                  role: 'assistant',
+                  content: text,
                   summarized: false,
                 },
               ];
@@ -295,22 +295,22 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
             };
 
             homeDispatch({
-              field: "selectedConversation",
+              field: 'selectedConversation',
               value: updatedConversation,
             });
           } else {
             let updatedMessages: Message[] = updatedConversation.messages.map(
               (messages, index) => {
                 if (index === updatedConversation.messages.length - 1) {
-                  if (message.content === "continue") {
+                  if (message.content === 'continue') {
                     let lastMessage =
                       updatedConversation.messages[
                         updatedConversation.messages.length - 1
                       ];
 
                     let updatedLastMessage = {
                       ...lastMessage,
-                      content: lastMessage.content + chunkValue,
+                      content: lastMessage.content + text,
                     };
 
                     return updatedLastMessage;
@@ -322,7 +322,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
                   }
                 }
                 return messages;
-              }
+              },
             );
 
             updatedConversation = {
@@ -331,7 +331,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
             };
 
             homeDispatch({
-              field: "selectedConversation",
+              field: 'selectedConversation',
               value: updatedConversation,
             });
           }

diff --git a/renderer/context/ModelSelection.tsx b/renderer/context/ModelSelection.tsx
@@ -120,6 +120,49 @@ const ModelProvider = ({ children }: { children: React.ReactNode }) => {
       });
   }, []);
 
+    useEffect(() => {
+    if (localServer.serverStatus === 'stopped') return;
+
+    let controller = new AbortController();
+
+    const getInferenceProcessHealth = async () => {
+      try {
+        const response = await fetch('http://127.0.0.1:8887/health', {
+          signal: controller.signal,
+        });
+
+        if (!response.ok) {
+          console.log('getInferenceProcessHealth error', response.status);
+        }
+
+        const healthData = await response.json();
+
+        if (healthData.status === 'ok' && localServer.serverStatus === 'loading') {
+          setModelLoaded(true);
+          setLocalServer({
+            serverStatus: "running",
+            serverMessage: "Local server is running",
+            model: selectedModel.id,
+          });
+        }
+      } catch (e: any) {
+        console.log(e);
+      }
+    }
+
+    getInferenceProcessHealth();
+    const inferenceProcessHealthInterval = setInterval(() => {
+      controller.abort();
+      controller = new AbortController();
+      getInferenceProcessHealth();
+    }, 5000);
+
+    return () => {
+      clearInterval(inferenceProcessHealthInterval);
+      controller.abort();
+    }
+  }, [localServer]);
+
   useEffect(() => {
     socket &&
       socket.on("model_loaded", () => {

diff --git a/src/index.ts b/src/index.ts
@@ -18,7 +18,6 @@ import { parse } from 'url';
 import machineUuid from 'machine-uuid';
 import util from 'util';
 import path from 'path';
-import { Readable } from 'stream';
 
 export let inferenceProcess: import('child_process').ChildProcessWithoutNullStreams =
   null as any;
@@ -234,48 +233,6 @@ const askForMediaAccess = async () => {
   return false;
 };
 
-const chat = async ({ data, endpoint = 'completion' }) => {
-  const encoder = new TextEncoder();
-  const stream = new Readable({
-    read() {},
-  });
-
-  const fetchStreamData = async () => {
-    const result = await fetch(
-      `http://127.0.0.1:${LLAMA_SERVER_PORT}/${endpoint}`,
-      {
-        method: 'POST',
-        body: JSON.stringify(data),
-        headers: {
-          'Content-Type': 'application/json',
-          'Access-Control-Allow-Origin': '*',
-        },
-      }
-    );
-
-    for await (const chunk of result.body as any) {
-      const t = Buffer.from(chunk).toString('utf8');
-
-      try {
-        if (t.startsWith('data: ')) {
-          const message = JSON.parse(t.substring(6));
-          (stream as any).push(encoder.encode(message.content));
-
-          if (message.stop) {
-            (stream as any).push(null);
-          }
-        }
-      } catch (error) {
-        (stream as any).push(null);
-      }
-    }
-  };
-
-  fetchStreamData();
-
-  return stream;
-};
-
 app.on('ready', () => {
   log.info('app event: ready');
   createWindow();
@@ -303,31 +260,10 @@ localServer.listen(LOCAL_SERVER_PORT, () => {
   log.info(`Server listening on port ${LOCAL_SERVER_PORT}`);
 });
 
-localServerApp.post('/api/edge', async (req, res) => {
-  const { endpoint, data } = req.body;
-
-  try {
-    try {
-      const streamResponse = await chat({
-        data,
-        endpoint
-      });
-
-      res.set({ 'Content-Type': 'text/plain' });
-      streamResponse.pipe(res);
-    } catch (error) {
-      log.error('/api/edge error (1)', error);
-      res.status(500).send('Something went wrong');
-    }
-  } catch (error) {
-    log.error('/api/edge error (2)', error);
-    res.status(500).send(`Something went wrong: ${error.message}`);
-  }
-});
-
 io.on('connection', (socket) => {
   log.info('socket connected');
 
+  socket.emit('LLAMA_SERVER_PORT', LLAMA_SERVER_PORT);
   machineUuid().then((uuid: string) => {
     socket.emit('machine_id', uuid);
   });

diff --git a/src/ports.ts b/src/ports.ts
@@ -1,3 +1,3 @@
 export const LOCAL_SERVER_PORT = 8888;
-export const LLAMA_SERVER_PORT = "8887";
+export const LLAMA_SERVER_PORT = 8887;
 export const OFFLINE_APP_PORT = 8886;