From 133628c65116233e288b79b6b3141f53d534613b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20=C3=87elik?= Date: Tue, 24 Sep 2024 22:08:17 +0300 Subject: [PATCH 1/2] fix: add custom abort controller --- ...y-request.ts => any-streamable-request.ts} | 2 +- examples/abort/custom-abort-controller.ts | 31 +++++++++++++++++++ ...uest.ts => specific-streamable-request.ts} | 0 src/browser.ts | 7 +++-- src/interfaces.ts | 25 +++++++++------ 5 files changed, 51 insertions(+), 14 deletions(-) rename examples/abort/{any-request.ts => any-streamable-request.ts} (93%) create mode 100644 examples/abort/custom-abort-controller.ts rename examples/abort/{specific-request.ts => specific-streamable-request.ts} (100%) diff --git a/examples/abort/any-request.ts b/examples/abort/any-streamable-request.ts similarity index 93% rename from examples/abort/any-request.ts rename to examples/abort/any-streamable-request.ts index a553ffb..a8f6c1a 100644 --- a/examples/abort/any-request.ts +++ b/examples/abort/any-streamable-request.ts @@ -4,7 +4,7 @@ import ollama from 'ollama' setTimeout(() => { console.log('\nAborting request...\n') ollama.abort() -}, 1000) // 1000 milliseconds = 1 second +}, 500) // 1000 milliseconds = 1 second ollama.generate({ model: 'llama3.1', diff --git a/examples/abort/custom-abort-controller.ts b/examples/abort/custom-abort-controller.ts new file mode 100644 index 0000000..fb2f05f --- /dev/null +++ b/examples/abort/custom-abort-controller.ts @@ -0,0 +1,31 @@ +import ollama from 'ollama' + +const abortController = new AbortController() + +// Set a timeout to abort the request after 100 milliseconds before the stream starts +setTimeout(() => { + console.log('\nAborting request...\n') + abortController.abort() +}, 100) // 100 milliseconds + + +ollama.generate({ + model: 'llama3.1', + prompt: 'Write a long story', + stream: true, // this could be a non-streamable request + abortController + }).then( + async (stream) => { + for await (const chunk of stream) { + process.stdout.write(chunk.response) + } + } + ).catch( + (error) => { + if (error.name === 'AbortError') { + console.log('The request has been aborted') + } else { + console.error('An error occurred:', error) + } + } + ) diff --git a/examples/abort/specific-request.ts b/examples/abort/specific-streamable-request.ts similarity index 100% rename from examples/abort/specific-request.ts rename to examples/abort/specific-streamable-request.ts diff --git a/src/browser.ts b/src/browser.ts index 7fa23cd..c5c4ba6 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -73,7 +73,7 @@ export class Ollama { if (request.stream) { const abortController = new AbortController() const response = await post(this.fetch, host, request, { - signal: abortController.signal, + signal: request.abortController?.signal ?? abortController.signal, headers: this.config.headers }) @@ -83,7 +83,7 @@ export class Ollama { const itr = parseJSON(response.body) const abortableAsyncIterator = new AbortableAsyncIterator( - abortController, + request.abortController ?? abortController, itr, () => { const i = this.ongoingStreamedRequests.indexOf(abortableAsyncIterator) @@ -96,7 +96,8 @@ export class Ollama { return abortableAsyncIterator } const response = await utils.post(this.fetch, host, request, { - headers: this.config.headers + headers: this.config.headers, + signal: request.abortController?.signal, }) return await response.json() } diff --git a/src/interfaces.ts b/src/interfaces.ts index 5b7b115..661e7ea 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -44,7 +44,11 @@ export interface Options { stop: string[] } -export interface GenerateRequest { +interface AbortableRequest { + abortController?: AbortController +} + +export interface GenerateRequest extends AbortableRequest{ model: string prompt: string suffix?: string @@ -95,7 +99,7 @@ export interface Tool { }; } -export interface ChatRequest { +export interface ChatRequest extends AbortableRequest { model: string messages?: Message[] stream?: boolean @@ -104,21 +108,22 @@ export interface ChatRequest { tools?: Tool[] options?: Partial + abortCtrl?: AbortController } -export interface PullRequest { +export interface PullRequest extends AbortableRequest { model: string insecure?: boolean stream?: boolean } -export interface PushRequest { +export interface PushRequest extends AbortableRequest { model: string insecure?: boolean stream?: boolean } -export interface CreateRequest { +export interface CreateRequest extends AbortableRequest { model: string path?: string modelfile?: string @@ -126,23 +131,23 @@ export interface CreateRequest { stream?: boolean } -export interface DeleteRequest { +export interface DeleteRequest extends AbortableRequest { model: string } -export interface CopyRequest { +export interface CopyRequest extends AbortableRequest { source: string destination: string } -export interface ShowRequest { +export interface ShowRequest extends AbortableRequest { model: string system?: string template?: string options?: Partial } -export interface EmbedRequest { +export interface EmbedRequest extends AbortableRequest { model: string input: string | string[] truncate?: boolean @@ -151,7 +156,7 @@ export interface EmbedRequest { options?: Partial } -export interface EmbeddingsRequest { +export interface EmbeddingsRequest extends AbortableRequest { model: string prompt: string keep_alive?: string | number From 724e9a28f219633d163ce666b8282577bf6c3023 Mon Sep 17 00:00:00 2001 From: bariscelik Date: Tue, 1 Oct 2024 16:19:59 +0300 Subject: [PATCH 2/2] chore: remove unnecessary abortCtrl --- src/interfaces.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/interfaces.ts b/src/interfaces.ts index 661e7ea..bf62401 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -108,7 +108,6 @@ export interface ChatRequest extends AbortableRequest { tools?: Tool[] options?: Partial - abortCtrl?: AbortController } export interface PullRequest extends AbortableRequest {