diff --git a/eng/test/mock_transport/src/mock_request.rs b/eng/test/mock_transport/src/mock_request.rs index dc193830e5..16f39581af 100644 --- a/eng/test/mock_transport/src/mock_request.rs +++ b/eng/test/mock_transport/src/mock_request.rs @@ -138,6 +138,7 @@ impl<'a> Serialize for RequestSerializer<'a> { Body::Bytes(bytes) => base64::encode(bytes as &[u8]), #[cfg(not(target_arch = "wasm32"))] Body::SeekableStream(_) => unimplemented!(), + Body::Multipart(_) => unimplemented!(), }, )?; diff --git a/eng/test/mock_transport/src/player_policy.rs b/eng/test/mock_transport/src/player_policy.rs index c72abadb1e..954fb59ebf 100644 --- a/eng/test/mock_transport/src/player_policy.rs +++ b/eng/test/mock_transport/src/player_policy.rs @@ -128,12 +128,14 @@ impl Policy for MockTransportPlayerPolicy { Body::Bytes(bytes) => bytes as &[u8], #[cfg(not(target_arch = "wasm32"))] Body::SeekableStream(_) => unimplemented!(), + Body::Multipart(_) => unimplemented!(), }; let expected_body = match expected_request.body() { Body::Bytes(bytes) => bytes as &[u8], #[cfg(not(target_arch = "wasm32"))] Body::SeekableStream(_) => unimplemented!(), + Body::Multipart(_) => unimplemented!(), }; if actual_body != expected_body { diff --git a/sdk/core/Cargo.toml b/sdk/core/Cargo.toml index 26413ef837..19510131d4 100644 --- a/sdk/core/Cargo.toml +++ b/sdk/core/Cargo.toml @@ -23,7 +23,7 @@ http-types = { version = "2.12", default-features = false } tracing = "0.1.40" rand = "0.8" reqwest = { version = "0.12.0", features = [ - "stream", + "stream", "multipart" ], default-features = false, optional = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -57,6 +57,7 @@ default = [] enable_reqwest = ["reqwest/default-tls"] enable_reqwest_gzip = ["reqwest/gzip"] enable_reqwest_rustls = ["reqwest/rustls-tls"] +enable_reqwest_multipart = ["reqwest/multipart"] hmac_rust = ["dep:sha2", "dep:hmac"] hmac_openssl = ["dep:openssl"] test_e2e = [] diff --git a/sdk/core/src/error/mod.rs b/sdk/core/src/error/mod.rs index e743a69656..716a668517 100644 --- a/sdk/core/src/error/mod.rs +++ b/sdk/core/src/error/mod.rs @@ -82,8 +82,10 @@ impl Display for ErrorKind { } } +// TODO: being able to derive PartialEq here would simplify some tests. +// Context::Custom inner boxed type prevents this. Check if there is a path to being able to derive it. /// An error encountered from interfacing with Azure -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct Error { context: Context, } @@ -383,7 +385,7 @@ where } } -#[derive(Debug)] +#[derive(Debug, PartialEq)] enum Context { Simple(ErrorKind), Message { @@ -400,6 +402,14 @@ struct Custom { error: Box, } +// Horrendous hack, so that I can easily assert_eq in tests. This makes the azure_core::Error unfortunately not implement +// PartialEq, which is necessary to make comparison and assertion outputs more readable. +impl PartialEq for Custom { + fn eq(&self, other: &Self) -> bool { + self.kind == other.kind && self.error.to_string() == other.error.to_string() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/sdk/core/src/headers/mod.rs b/sdk/core/src/headers/mod.rs index 7529a59e8d..55850ee7ad 100644 --- a/sdk/core/src/headers/mod.rs +++ b/sdk/core/src/headers/mod.rs @@ -166,11 +166,13 @@ impl Debug for Headers { let redacted = HeaderValue::from_static("[redacted]"); f.debug_map() .entries(self.0.iter().map(|(name, value)| { - if matching_ignore_ascii_case(name.as_str(), AUTHORIZATION.as_str()) { - (name, value) - } else { - (name, &redacted) - } + (name, value) + // TODO: restore + // if matching_ignore_ascii_case(name.as_str(), AUTHORIZATION.as_str()) { + // (name, value) + // } else { + // (name, &redacted) + // } })) .finish()?; write!(f, ")") diff --git a/sdk/core/src/http_client/mod.rs b/sdk/core/src/http_client/mod.rs index a531493d46..83b9499090 100644 --- a/sdk/core/src/http_client/mod.rs +++ b/sdk/core/src/http_client/mod.rs @@ -12,6 +12,7 @@ use async_trait::async_trait; use bytes::Bytes; use serde::{de::DeserializeOwned, Serialize}; use std::sync::Arc; +use crate::to_reqwest_form; /// Construct a new `HttpClient` pub fn new_http_client() -> Arc { diff --git a/sdk/core/src/http_client/reqwest.rs b/sdk/core/src/http_client/reqwest.rs index 12d892fa69..af64f706a1 100644 --- a/sdk/core/src/http_client/reqwest.rs +++ b/sdk/core/src/http_client/reqwest.rs @@ -17,6 +17,7 @@ pub fn new_reqwest_client() -> Arc { // See for more details. #[cfg(not(target_arch = "wasm32"))] let client = ::reqwest::ClientBuilder::new() + .connection_verbose(true) .pool_max_idle_per_host(0) .build() .expect("failed to build `reqwest` client"); @@ -43,6 +44,7 @@ impl HttpClient for ::reqwest::Client { let body = request.body().clone(); let reqwest_request = match body { + Body::Multipart(form) => req.multipart(super::to_reqwest_form(form)).build(), Body::Bytes(bytes) => req.body(bytes).build(), // We cannot currently implement `Body::SeekableStream` for WASM diff --git a/sdk/core/src/request.rs b/sdk/core/src/request.rs index c8e563c27e..d2babf44ae 100644 --- a/sdk/core/src/request.rs +++ b/sdk/core/src/request.rs @@ -11,6 +11,7 @@ use std::fmt::Debug; /// An HTTP Body. #[derive(Debug, Clone)] pub enum Body { + Multipart(MyForm), /// A body of a known size. Bytes(bytes::Bytes), /// A streaming body. @@ -24,6 +25,7 @@ pub enum Body { impl Body { pub fn len(&self) -> usize { match self { + Body::Multipart(_) => 0, Body::Bytes(bytes) => bytes.len(), #[cfg(not(target_arch = "wasm32"))] Body::SeekableStream(stream) => stream.len(), @@ -36,6 +38,7 @@ impl Body { pub(crate) async fn reset(&mut self) -> crate::Result<()> { match self { + Body::Multipart(_) => Ok(()), Body::Bytes(_) => Ok(()), #[cfg(not(target_arch = "wasm32"))] Body::SeekableStream(stream) => stream.reset().await, @@ -59,6 +62,12 @@ impl From> for Body { } } +impl From for Body { + fn from(my_form: MyForm) -> Self { + Self::Multipart(my_form) + } +} + /// A pipeline request. /// /// A pipeline request is composed by a destination (uri), a method, a collection of headers and a @@ -129,6 +138,10 @@ impl Request { self.body = body.into(); } + pub fn multipart(&mut self, form: MyForm) { + self.body = Body::Multipart(form); + } + pub fn insert_header(&mut self, key: K, value: V) where K: Into, @@ -147,3 +160,59 @@ impl Request { self.insert_header(item.name(), item.value()); } } + +// Had to add this type because reqwest::multipart::Form does not implement Clone +// reqwest seems to handle the calculation of the content-size, so we don't need to keep +// track of that here. In a proper implementation, we might need to handle it. +#[derive(Debug, Clone)] +pub struct MyForm { + pub(crate) parts: Vec, +} + +impl MyForm { + pub fn new() -> Self { + Self { parts: Vec::new() } + } + + pub fn text(mut self, name: impl Into, value: impl Into) -> Self { + self.parts.push(MyPart::Text { + name: name.into(), + value: value.into(), + }); + self + } + + pub fn file(mut self, name: impl Into, bytes: Vec) -> Self { + self.parts.push(MyPart::File { + name: name.into(), + bytes: bytes, + }); + self + } +} + +#[derive(Debug, Clone)] +pub enum MyPart { + Text{name: String, value: String}, + File{name: String, bytes: Vec}, +} + +pub(crate) fn to_reqwest_form(form: MyForm) -> reqwest::multipart::Form { + let mut reqwest_form = reqwest::multipart::Form::new(); + for part in form.parts { + match part { + MyPart::Text { name, value } => { + reqwest_form = reqwest_form.text(name, value); + } + // "part name" is no the same as `file_name`. Learned the hard way... + MyPart::File { name, bytes } => { + reqwest_form = reqwest_form.part("file", + reqwest::multipart::Part::bytes(bytes). + mime_str("application/octet-stream").unwrap() + .file_name(name) + ); + } + } + } + reqwest_form +} diff --git a/sdk/data_tables/src/transaction.rs b/sdk/data_tables/src/transaction.rs index 621eafec56..d26f15fcc8 100644 --- a/sdk/data_tables/src/transaction.rs +++ b/sdk/data_tables/src/transaction.rs @@ -62,6 +62,7 @@ impl TransactionOperations { } #[cfg(not(target_arch = "wasm32"))] azure_core::Body::SeekableStream(_) => todo!(), + azure_core::Body::Multipart(_) => todo!(), } } diff --git a/sdk/openai_inference/Cargo.toml b/sdk/openai_inference/Cargo.toml new file mode 100644 index 0000000000..536fa69738 --- /dev/null +++ b/sdk/openai_inference/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "azure_openai_inference" +version = "0.1.0" +description = "Rust wrappers around Microsoft Azure REST APIs - Azure OpenAI Inference" +readme = "README.md" +authors = ["Microsoft Corp."] +license = "MIT" +repository = "https://github.com/azure/azure-sdk-for-rust" +homepage = "https://github.com/azure/azure-sdk-for-rust" +# documentation = "https://docs.rs/azure_data_cosmos" +keywords = ["sdk", "azure", "rest"] +categories = ["api-bindings"] +edition = "2021" + +[dependencies] +async-trait = "0.1" +azure_core = { path = "../core", version = "0.20" } +time = "0.3.10" +futures = "0.3" +tracing = "0.1.40" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +url = "2.2" +# uuid = { version = "1.0", features = ["v4"] } +thiserror = "1.0" +bytes = "1.0" +log = "0.4" +env_logger = "0.10" + +[dev-dependencies] +# tracing-subscriber = "0.3" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +# clap = { version = "4.0.2", features = ["derive", "env"] } +reqwest = "0.12.0" +# stop-token = { version = "0.7.0", features = ["tokio"] } +mock_transport = { path = "../../eng/test/mock_transport" } + +[features] +default = ["enable_reqwest", "hmac_rust"] +enable_reqwest = ["azure_core/enable_reqwest"] +enable_reqwest_rustls = ["azure_core/enable_reqwest_rustls"] +test_e2e = [] +hmac_rust = ["azure_core/hmac_rust"] +hmac_openssl = ["azure_core/hmac_openssl"] + +[package.metadata.docs.rs] +features = ["enable_reqwest", "enable_reqwest_rustls", "hmac_rust", "hmac_openssl"] diff --git a/sdk/openai_inference/README.md b/sdk/openai_inference/README.md new file mode 100644 index 0000000000..c3b8434396 --- /dev/null +++ b/sdk/openai_inference/README.md @@ -0,0 +1 @@ +# azure_openai_inference diff --git a/sdk/openai_inference/assets/JP_it_is_rainy_today.wav b/sdk/openai_inference/assets/JP_it_is_rainy_today.wav new file mode 100644 index 0000000000..5970c85ec1 Binary files /dev/null and b/sdk/openai_inference/assets/JP_it_is_rainy_today.wav differ diff --git a/sdk/openai_inference/assets/batman.wav b/sdk/openai_inference/assets/batman.wav new file mode 100644 index 0000000000..4c0b7248a3 Binary files /dev/null and b/sdk/openai_inference/assets/batman.wav differ diff --git a/sdk/openai_inference/examples/chat_completions_azure.rs b/sdk/openai_inference/examples/chat_completions_azure.rs new file mode 100644 index 0000000000..6719a1b1f7 --- /dev/null +++ b/sdk/openai_inference/examples/chat_completions_azure.rs @@ -0,0 +1,35 @@ +use azure_openai_inference::{AzureKeyCredential, AzureOpenAIClient}; +use azure_openai_inference::{AzureServiceVersion, CreateChatCompletionsRequest}; + +#[tokio::main] +async fn main() { + let endpoint = + std::env::var("AZURE_OPENAI_ENDPOINT").expect("Set AZURE_OPENAI_ENDPOINT env variable"); + let secret = std::env::var("AZURE_OPENAI_KEY").expect("Set AZURE_OPENAI_KEY env variable"); + + let openai_client = AzureOpenAIClient::new(endpoint, AzureKeyCredential::new(secret)); + + let chat_completions_request = CreateChatCompletionsRequest::new_with_user_message( + "gpt-4-1106-preview", + "Tell me a joke about pineapples", + ); + + println!("{:#?}", &chat_completions_request); + println!("{:#?}", serde_json::to_string(&chat_completions_request)); + let response = openai_client + .create_chat_completions( + &chat_completions_request.model, + AzureServiceVersion::V2023_12_01Preview, + &chat_completions_request, + ) + .await; + + match response { + Ok(chat_completions) => { + println!("{:#?}", &chat_completions); + } + Err(e) => { + println!("Error: {}", e); + } + } +} diff --git a/sdk/openai_inference/examples/chat_completions_non_azure.rs b/sdk/openai_inference/examples/chat_completions_non_azure.rs new file mode 100644 index 0000000000..6bab06db74 --- /dev/null +++ b/sdk/openai_inference/examples/chat_completions_non_azure.rs @@ -0,0 +1,30 @@ +use azure_openai_inference::CreateChatCompletionsRequest; +use azure_openai_inference::OpenAIClient; +use azure_openai_inference::OpenAIKeyCredential; + +#[tokio::main] +async fn main() { + let secret = + std::env::var("NON_AZURE_OPENAI_KEY").expect("Set NON_AZURE_OPENAI_KEY env variable"); + + let openai_client = OpenAIClient::new(OpenAIKeyCredential::new(secret)); + + let chat_completions_request = CreateChatCompletionsRequest::new_with_user_message( + "gpt-3.5-turbo-1106", + "Tell me a joke about pineapples", + ); + + println!("{:#?}", &chat_completions_request); + let response = openai_client + .create_chat_completions(&chat_completions_request) + .await; + + match response { + Ok(chat_completions) => { + println!("{:#?}", &chat_completions); + } + Err(e) => { + println!("Error: {}", e); + } + } +} diff --git a/sdk/openai_inference/examples/chat_completions_streaming_azure.rs b/sdk/openai_inference/examples/chat_completions_streaming_azure.rs new file mode 100644 index 0000000000..f45e5d70e5 --- /dev/null +++ b/sdk/openai_inference/examples/chat_completions_streaming_azure.rs @@ -0,0 +1,52 @@ +use std::io::{self, Write}; + +use azure_core::Result; + +use azure_openai_inference::{AzureKeyCredential, AzureOpenAIClient}; +use azure_openai_inference::{AzureServiceVersion, CreateChatCompletionsRequest}; +use futures::stream::StreamExt; + +#[tokio::main] +async fn main() -> Result<()> { + let endpoint = + std::env::var("AZURE_OPENAI_ENDPOINT").expect("Set AZURE_OPENAI_ENDPOINT env variable"); + let secret = std::env::var("AZURE_OPENAI_KEY").expect("Set AZURE_OPENAI_KEY env variable"); + + let openai_client = AzureOpenAIClient::new(endpoint, AzureKeyCredential::new(secret)); + + let chat_completions_request = CreateChatCompletionsRequest::new_stream_with_user_message( + "gpt-4-1106-preview", + "Write me an essay that is at least 200 words long on the nutritional values (or lack thereof) of fast food. + Start the essay by stating 'this essay will be x many words long' where x is the number of words in the essay.",); + + println!("{:#?}", &chat_completions_request); + println!("{:#?}", serde_json::to_string(&chat_completions_request)); + let response = openai_client + .stream_chat_completion( + &chat_completions_request.model, + AzureServiceVersion::V2023_12_01Preview, + &chat_completions_request, + ) + .await?; + + // this pins the stream to the stack so it is safe to poll it (namely, it won't be dealloacted or moved) + futures::pin_mut!(response); + + while let Some(result) = response.next().await { + match result { + Ok(delta) => { + if let Some(choice) = delta.choices.get(0) { + choice.delta.as_ref().map(|d| { + d.content.as_ref().map(|c| { + print!("{}", c); + io::stdout().flush(); + }); + }); + } + } + Err(e) => println!("Error: {:?}", e), + } + } + + Ok(()) +} diff --git a/sdk/openai_inference/examples/speech_transcription_azure.rs b/sdk/openai_inference/examples/speech_transcription_azure.rs new file mode 100644 index 0000000000..f25d755da9 --- /dev/null +++ b/sdk/openai_inference/examples/speech_transcription_azure.rs @@ -0,0 +1,44 @@ +use std::fs::File; +use std::io::Read; + +use azure_openai_inference::{AzureKeyCredential, AzureOpenAIClient}; +use azure_openai_inference::{AzureServiceVersion, CreateTranscriptionRequest}; +use env_logger::Env; + +#[tokio::main] +async fn main() { + // use `RUST_LOG=reqwest=trace,hyper=trace cargo run --example speech_transcription_azure` to get request traces + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + + let endpoint = + std::env::var("AZURE_OPENAI_ENDPOINT").expect("Set AZURE_OPENAI_ENDPOINT env variable"); + let secret = std::env::var("AZURE_OPENAI_KEY").expect("Set AZURE_OPENAI_KEY env variable"); + + let openai_client = AzureOpenAIClient::new(endpoint, AzureKeyCredential::new(secret)); + + let mut file = File::open("./sdk/openai_inference/assets/JP_it_is_rainy_today.wav") + .expect("File not found"); + let mut file_contents = Vec::new(); + let _ = file + .read_to_end(&mut file_contents) + .expect("Failed to read file"); + + let create_transcription_request = + CreateTranscriptionRequest::new_as_text(file_contents, "JP_it_is_rainy_today.wav"); + let response = openai_client + .create_speech_transcription( + "whisper-deployment", + AzureServiceVersion::V2023_12_01Preview, + &create_transcription_request, + ) + .await; + + match response { + Ok(transcription) => { + println!("{:#?}", &transcription); + } + Err(e) => { + println!("Error: {}", e); + } + } +} diff --git a/sdk/openai_inference/examples/speech_transcription_non_azure.rs b/sdk/openai_inference/examples/speech_transcription_non_azure.rs new file mode 100644 index 0000000000..16bac71b48 --- /dev/null +++ b/sdk/openai_inference/examples/speech_transcription_non_azure.rs @@ -0,0 +1,39 @@ +use std::fs::File; +use std::io::Read; + +use azure_openai_inference::CreateTranscriptionRequest; +use azure_openai_inference::{OpenAIClient, OpenAIKeyCredential}; +use env_logger::Env; + +#[tokio::main] +async fn main() { + // use `RUST_LOG=reqwest=trace,hyper=trace cargo run --example speech_transcription_azure` to get request traces + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + + let secret = std::env::var("OPENAI_KEY").expect("Set AZURE_OPENAI_KEY env variable"); + + let openai_client = OpenAIClient::new(OpenAIKeyCredential::new(secret)); + + let mut file = File::open("./sdk/openai_inference/assets/batman.wav").expect("File not found"); + let mut file_contents = Vec::new(); + let _ = file + .read_to_end(&mut file_contents) + .expect("Failed to read file"); + + let create_transcription_request = + CreateTranscriptionRequest::new_as_text(file_contents, "batman.wav"); + + println!("{:#?}", &create_transcription_request.model); + let response = openai_client + .create_speech_transcription(&create_transcription_request) + .await; + + match response { + Ok(transcription) => { + println!("{:#?}", &transcription); + } + Err(e) => { + println!("Error: {}", e); + } + } +} diff --git a/sdk/openai_inference/src/auth/mod.rs b/sdk/openai_inference/src/auth/mod.rs new file mode 100644 index 0000000000..e03d485fca --- /dev/null +++ b/sdk/openai_inference/src/auth/mod.rs @@ -0,0 +1,41 @@ +use azure_core::{ + auth::Secret, + headers::{HeaderName, HeaderValue, AUTHORIZATION}, + Header, +}; + +pub struct AzureKeyCredential(Secret); + +pub struct OpenAIKeyCredential(Secret); + +impl OpenAIKeyCredential { + pub fn new(access_token: String) -> Self { + Self(Secret::new(access_token)) + } +} + +impl AzureKeyCredential { + pub fn new(api_key: String) -> Self { + Self(Secret::new(api_key)) + } +} + +impl Header for AzureKeyCredential { + fn name(&self) -> HeaderName { + HeaderName::from_static("api-key") + } + + fn value(&self) -> HeaderValue { + HeaderValue::from_cow(format!("{}", self.0.secret())) + } +} + +impl Header for OpenAIKeyCredential { + fn name(&self) -> HeaderName { + AUTHORIZATION + } + + fn value(&self) -> HeaderValue { + HeaderValue::from_cow(format!("Bearer {}", &self.0.secret())) + } +} diff --git a/sdk/openai_inference/src/clients/azure_openai.rs b/sdk/openai_inference/src/clients/azure_openai.rs new file mode 100644 index 0000000000..5d54a4a9ff --- /dev/null +++ b/sdk/openai_inference/src/clients/azure_openai.rs @@ -0,0 +1,184 @@ +use std::sync::Arc; + +use crate::{ + AzureKeyCredential, CreateChatCompletionsRequest, CreateChatCompletionsResponse, + CreateChatCompletionsStreamResponse, CreateTranscriptionRequest, +}; +use azure_core::{Error, HttpClient, Method, MyForm, Result, Url}; +use futures::stream::TryStreamExt; +use futures::{stream, Stream, StreamExt}; + +use super::stream::{ChatCompletionStreamHandler, EventStreamer}; + +pub struct AzureOpenAIClient { + http_client: Arc, + endpoint: String, + key_credential: AzureKeyCredential, +} + +impl AzureOpenAIClient { + pub fn new(endpoint: String, key_credential: AzureKeyCredential) -> Self { + Self { + http_client: azure_core::new_http_client(), + endpoint, + key_credential, + } + } + + pub async fn create_chat_completions( + &self, + deployment_name: &str, + api_version: AzureServiceVersion, + chat_completions_request: &CreateChatCompletionsRequest, + ) -> Result { + let url = Url::parse(&format!( + "{}/openai/deployments/{}/chat/completions?api-version={}", + &self.endpoint, + deployment_name, + api_version.as_str() + ))?; + let request = super::build_request( + &self.key_credential, + url, + Method::Post, + chat_completions_request, + )?; + let response = self.http_client.execute_request(&request).await?; + response.json::().await + } + + // This works, it's a simple implementation of the streaming, no chunking + // pub async fn stream_chat_completion_raw_chunks(&self, deployment_name: &str, api_version: AzureServiceVersion, + // chat_completions_request: &CreateChatCompletionsRequest) + // -> Result>> { + // let url = Url::parse(&format!("{}/openai/deployments/{}/chat/completions?api-version={}", + // &self.endpoint, + // deployment_name, + // api_version.as_str()) + // )?; + // let request = super::build_request(&self.key_credential, url, Method::Post, chat_completions_request)?; + // let response = self.http_client.execute_request(&request).await?; + + // Ok(response.into_body() + // .and_then(|chunk| { + // std::future::ready(std::str::from_utf8(&chunk) + // .map(String::from) + // .map_err(Error::from) + // ) + // } + // )) + // } + + // method used just for generating test data + pub async fn stream_chat_completion_stdout_dump( + &self, + deployment_name: &str, + api_version: AzureServiceVersion, + chat_completions_request: &CreateChatCompletionsRequest, + ) -> Result<()> { + let url = Url::parse(&format!( + "{}/openai/deployments/{}/chat/completions?api-version={}", + &self.endpoint, + deployment_name, + api_version.as_str() + ))?; + let request = super::build_request( + &self.key_credential, + url, + Method::Post, + chat_completions_request, + )?; + let mut response = self.http_client.execute_request(&request).await?.into_body(); + + while let Some(chunk) = response.next().await { + match chunk { + Ok(chunk) => { + let string_chunk = std::str::from_utf8(&chunk).expect("String chunk into utf8 failure"); + println!("START OF NEW CHUNK"); + println!(); + println!(); + println!("{:?}", string_chunk); + println!(); + println!(); + println!("END OF CHUNK"); + println!(); + println!(); + + }, + Err(_) => todo!(), + } + } + + Ok(()) + } + + pub async fn stream_chat_completion( + &self, + deployment_name: &str, + api_version: AzureServiceVersion, + chat_completions_request: &CreateChatCompletionsRequest, + ) -> Result>> { + let url = Url::parse(&format!( + "{}/openai/deployments/{}/chat/completions?api-version={}", + &self.endpoint, + deployment_name, + api_version.as_str() + ))?; + let request = super::build_request( + &self.key_credential, + url, + Method::Post, + chat_completions_request, + )?; + let response = self.http_client.execute_request(&request).await?; + + let mut response_body = response.into_body(); + let stream_handler = ChatCompletionStreamHandler::new("\n\n"); + + let stream = stream_handler.event_stream(response_body).await?; + Ok(stream) + } + + pub async fn create_speech_transcription( + &self, + deployment_name: &str, + api_version: AzureServiceVersion, + create_transcription_request: &CreateTranscriptionRequest, + ) -> Result { + let url = Url::parse(&format!( + "{}/openai/deployments/{}/audio/transcriptions?api-version={}", + &self.endpoint, + deployment_name, + api_version.as_str() + ))?; + + let request = super::build_multipart_request(&self.key_credential, url, || { + Ok(MyForm::new() + .text( + "response_format", + create_transcription_request.response_format.to_string(), + ) + .file( + create_transcription_request.file_name.clone(), + create_transcription_request.file.clone(), + )) + }); + + let response = self.http_client.execute_request(&request?).await?; + Ok(response.into_body().collect_string().await?) + } +} + +pub enum AzureServiceVersion { + V2023_09_01Preview, + V2023_12_01Preview, +} + +impl AzureServiceVersion { + pub fn as_str(&self) -> &'static str { + match self { + AzureServiceVersion::V2023_09_01Preview => "2023-09-01-preview", + AzureServiceVersion::V2023_12_01Preview => "2023-12-01-preview", + } + } +} diff --git a/sdk/openai_inference/src/clients/mod.rs b/sdk/openai_inference/src/clients/mod.rs new file mode 100644 index 0000000000..a5cbdf08af --- /dev/null +++ b/sdk/openai_inference/src/clients/mod.rs @@ -0,0 +1,46 @@ +use azure_core::{ + headers::{ACCEPT, CONTENT_TYPE}, + Header, Method, MyForm, Request, +}; +use azure_core::{Result, Url}; +use serde::Serialize; + +pub mod azure_openai; +pub mod openai; +mod stream; +#[cfg(test)] +mod tests; + +pub(crate) fn build_request( + key_credential: &impl Header, + url: Url, + method: Method, + data: &T, +) -> Result +where + T: ?Sized + Serialize, +{ + let mut request = Request::new(url, method); + request.add_mandatory_header(key_credential); + request.insert_header(CONTENT_TYPE, "application/json"); + request.insert_header(ACCEPT, "application/json"); + request.set_json(data)?; + Ok(request) +} + +pub(crate) fn build_multipart_request( + key_credential: &impl Header, + url: Url, + form_generator: F, +) -> Result +where + F: FnOnce() -> Result, +{ + let mut request = Request::new(url, Method::Post); + request.add_mandatory_header(key_credential); + // handled insternally by reqwest + // request.insert_header(CONTENT_TYPE, "multipart/form-data"); + // request.insert_header(ACCEPT, "application/json"); + request.multipart(form_generator()?); + Ok(request) +} diff --git a/sdk/openai_inference/src/clients/openai.rs b/sdk/openai_inference/src/clients/openai.rs new file mode 100644 index 0000000000..f25103cfe7 --- /dev/null +++ b/sdk/openai_inference/src/clients/openai.rs @@ -0,0 +1,66 @@ +use std::sync::Arc; + +use azure_core::{HttpClient, Method, MyForm, Result, Url}; + +use crate::auth::OpenAIKeyCredential; +use crate::{ + CreateChatCompletionsRequest, CreateChatCompletionsResponse, CreateTranscriptionRequest, +}; + +pub struct OpenAIClient { + http_client: Arc, + key_credential: OpenAIKeyCredential, // should this be an Arc? Probably not, we want this live as long as the client +} + +impl OpenAIClient { + pub fn new(key_credential: OpenAIKeyCredential) -> Self { + Self { + http_client: azure_core::new_http_client(), + key_credential, + } + } + + pub async fn create_chat_completions( + &self, + chat_completions_request: &CreateChatCompletionsRequest, + ) -> Result { + let url = Url::parse("https://api.openai.com/v1/chat/completions")?; + let request = super::build_request( + &self.key_credential, + url, + Method::Post, + chat_completions_request, + )?; + let response = self.http_client.execute_request(&request).await?; + response.json::().await + } + + pub async fn create_speech_transcription( + &self, + create_transcription_request: &CreateTranscriptionRequest, + ) -> Result { + let url = Url::parse(&format!("https://api.openai.com/v1/audio/transcriptions"))?; + + let request = super::build_multipart_request(&self.key_credential, url, || { + Ok(MyForm::new() + .text( + "response_format", + create_transcription_request.response_format.to_string(), + ) + .text( + "model", + create_transcription_request + .model + .as_ref() + .expect("'model' is required"), + ) + .file( + create_transcription_request.file_name.clone(), + create_transcription_request.file.clone(), + )) + }); + + let response = self.http_client.execute_request(&request?).await?; + Ok(response.into_body().collect_string().await?) + } +} diff --git a/sdk/openai_inference/src/clients/stream.rs b/sdk/openai_inference/src/clients/stream.rs new file mode 100644 index 0000000000..7e75c37ad7 --- /dev/null +++ b/sdk/openai_inference/src/clients/stream.rs @@ -0,0 +1,308 @@ +use std::borrow::Borrow; +use std::pin::Pin; + +use async_trait::async_trait; +use azure_core::ResponseBody; +use azure_core::{Error, Result}; +use futures::{Stream, TryFutureExt}; +use futures::StreamExt; +use futures::TryStreamExt; +use tracing::debug; + +use crate::CreateChatCompletionsStreamResponse; + +#[async_trait::async_trait] +pub trait EventStreamer { + // read more on Higher-Rank Trait Bounds (HRTBs) + async fn event_stream<'a>( + &self, + mut response_body: ResponseBody, + ) -> Result> + 'a>>> + where + T: serde::de::DeserializeOwned + 'a; +} + +// there will be polymorphic streams where the along with a "data:" payload, there will be an "event:" payload +// implying a per-event deserialization type. Customer consumption needs to be as seemless as much as as possible. +pub struct ChatCompletionStreamHandler { + pub(crate) stream_event_delimiter: String, +} + +impl ChatCompletionStreamHandler { + + pub fn new(stream_event_delimiter: impl Into) -> Self { + ChatCompletionStreamHandler { + stream_event_delimiter: stream_event_delimiter.into() + } + } +} + +#[async_trait::async_trait] +impl EventStreamer for ChatCompletionStreamHandler { + async fn event_stream<'a>( + &self, + mut response_body: ResponseBody, + ) -> Result> + 'a>>> { + + let stream = string_chunks(response_body, &self.stream_event_delimiter).await? + .map_ok(|event| { + // println!("EVENT AS A STRING: {:?}", &event); + serde_json::from_str::(&event).expect("Deserialization failed") + // CreateChatCompletionsStreamResponse { choices: vec![] } + }); + Ok(Box::pin(stream)) + } +} + +/// This function chunks a response body from an HTTP request. It assumes a UTF8 encoding. The delimiter of chunks +/// can be different on whether it's an Azure endpoint or the unbranded OpenAI service. +/// +/// * `response_body` - The response body from an HTTP request. Using a type easy to test but hard to read. This is just a azure_core::ResponseBody +/// * `stream_event_delimiter` - The delimiter of server sent events. Usually either "\n\n" or "\r\n\r\n". +async fn string_chunks( + response_body: (impl Stream> + Unpin), + _stream_event_delimiter: &str, // figure out how to use it in the move +) -> Result>> { + let chunk_buffer = Vec::new(); + let stream = futures::stream::unfold( + (response_body, chunk_buffer), + |(mut response_body, mut chunk_buffer)| async move { + + + // Need to figure out a way how I can move the _stream_event_delimiter into this closure + let delimiter = b"\n\n"; + let delimiter_len = delimiter.len(); + + if let Some(Ok(bytes)) = response_body.next().await { + chunk_buffer.extend_from_slice(&bytes); + // Looking for the next occurence of the event delimiter + // it's + 4 because the \n\n are escaped and represented as [92, 110, 92, 110] + if let Some(pos) = chunk_buffer.windows(delimiter_len).position(|window| window == delimiter) { + // the range must include the delimiter bytes + let mut bytes = chunk_buffer.drain(..pos + delimiter_len).collect::>(); + bytes.truncate(bytes.len() - delimiter_len); + + return if let Ok(yielded_value) = std::str::from_utf8(&bytes) { + // We strip the "data: " portion of the event. The rest is always JSON and will be deserialized + // by a subsquent mapping function for this stream + let yielded_value = yielded_value.trim_start_matches("data:").trim(); + if yielded_value == "[DONE]" { + return None; + } else { + Some((Ok(yielded_value.to_string()), (response_body, chunk_buffer))) + } + } else { + None + }; + } + if chunk_buffer.len() > 0 { + return Some(( + Err(Error::with_message( + azure_core::error::ErrorKind::DataConversion, + || "Incomplete chunk", + )), + (response_body, chunk_buffer), + )); + } + // We drain the buffer of any messages that may be left over. + // The block above will be skipped, since response_body.next() will be None every time + } else if !chunk_buffer.is_empty() { + // we need to verify if there are any event left in the buffer and emit them individually + // it's + 4 because the \n\n are escaped and represented as [92, 110, 92, 110] + if let Some(pos) = chunk_buffer.windows(delimiter_len).position(|window| window == delimiter) { + // the range must include the delimiter bytes + let mut bytes = chunk_buffer.drain(..pos + delimiter_len).collect::>(); + bytes.truncate(bytes.len() - delimiter_len); + + return if let Ok(yielded_value) = std::str::from_utf8(&bytes) { + let yielded_value = yielded_value.trim_start_matches("data:").trim(); + if yielded_value == "[DONE]" { + return None; + } else { + Some((Ok(yielded_value.to_string()), (response_body, chunk_buffer))) + } + } else { + None + }; + } + // if we get to this point, it means we have drained the buffer of all events, meaning that we haven't been able to find the next delimiter + } + None + }, + ); + + // We filter errors, we should specifically target the error type yielded when we are not able to find an event in a chunk + // Specifically the Error::with_messagge(ErrorKind::DataConversion, || "Incomplete chunk") + return Ok(stream.filter(|it| std::future::ready(it.is_ok()))); +} + +#[cfg(test)] +mod tests { + use std::string; + + use crate::clients::tests::*; + + use super::*; + use azure_core::ResponseBody; + use futures::pin_mut; + use tracing::debug; + + #[tokio::test] + async fn clean_chunks() -> Result<()> { + let mut source_stream = futures::stream::iter(vec![ + Ok(bytes::Bytes::from_static(b"data: piece 1\\n\\n")), + Ok(bytes::Bytes::from_static(b"data: piece 2\\n\\n")), + Ok(bytes::Bytes::from_static(b"data: [DONE]\\n\\n")), + ]); + + let actual = string_chunks(&mut source_stream, "\\n\\n").await?; + let actual: Vec> = actual.collect().await; + + let expected: Vec> = + vec![Ok("piece 1".to_string()), Ok("piece 2".to_string())]; + assert_eq!(expected, actual); + + Ok(()) + } + + #[tokio::test] + async fn multiple_message_in_one_chunk() -> Result<()> { + let mut source_stream = futures::stream::iter(vec![ + Ok(bytes::Bytes::from_static(b"data: piece 1\\n\\ndata: piece 2\\n\\n")), + Ok(bytes::Bytes::from_static(b"data: piece 3\\n\\ndata: [DONE]\\n\\n")), + ]); + + let mut actual = Vec::new(); + + let actual_stream = string_chunks(&mut source_stream, "\n\n").await?; + pin_mut!(actual_stream); + + while let Some(event) = actual_stream.next().await { + actual.push(event); + } + + let expected: Vec> = vec![ + Ok("piece 1".to_string()), + Ok("piece 2".to_string()), + Ok("piece 3".to_string()), + ]; + assert_eq!(expected, actual); + Ok(()) + } + + #[tokio::test] + async fn data_marker_in_previous_chunk() -> Result<()> { + let mut source_stream = futures::stream::iter(vec![ + Ok(bytes::Bytes::from_static(b"data: piece 1\\n\\ndata: piece 2\\n\\ndata:")), + Ok(bytes::Bytes::from_static(b" piece 3\\n\\ndata: [DONE]\\n\\n")), + ]); + + let mut actual = Vec::new(); + + let actual_stream = string_chunks(&mut source_stream, "\n\n").await?; + pin_mut!(actual_stream); + + while let Some(event) = actual_stream.next().await { + actual.push(event); + } + + let expected: Vec> = vec![ + Ok("piece 1".to_string()), + Ok("piece 2".to_string()), + Ok("piece 3".to_string()), + ]; + assert_eq!(expected, actual); + Ok(()) + } + + #[tokio::test] + async fn event_delimeter_split_across_chunks() -> Result<()> { + let mut source_stream = futures::stream::iter(vec![ + Ok(bytes::Bytes::from_static(b"data: piece 1\\n")), + Ok(bytes::Bytes::from_static(b"\\ndata: [DONE]")), + ]); + + let actual = string_chunks(&mut source_stream, "\n\n").await?; + let actual: Vec> = actual.collect().await; + + let expected: Vec> = vec![ + Ok("piece 1".to_string()), + ]; + assert_eq!(expected, actual); + Ok(()) + } + + #[tokio::test] + async fn event_delimiter_at_start_of_next_chunk() -> Result<()> { + let mut source_stream = futures::stream::iter(vec![ + Ok(bytes::Bytes::from_static(b"data: piece 1")), + Ok(bytes::Bytes::from_static(b"\\n\\ndata: [DONE]")), + ]); + + let actual = string_chunks(&mut source_stream, "\n\n").await?; + let actual: Vec> = actual.collect().await; + + let expected: Vec> = vec![ + Ok("piece 1".to_string()), + ]; + assert_eq!(expected, actual); + Ok(()) + } + + #[tokio::test] + async fn real_data() -> Result<()> { + let mut source_stream = futures::stream::iter(vec![ + Ok(bytes::Bytes::from(STREAM_CHUNK_01)), + Ok(bytes::Bytes::from(STREAM_CHUNK_02)), + Ok(bytes::Bytes::from(STREAM_CHUNK_03)), + // Ok(bytes::Bytes::from(STREAM_CHUNK_04)), + // Ok(bytes::Bytes::from(STREAM_CHUNK_05)), + // Ok(bytes::Bytes::from(STREAM_CHUNK_06)), + // Ok(bytes::Bytes::from(STREAM_CHUNK_07)), + // Ok(bytes::Bytes::from(STREAM_CHUNK_08)), + // Ok(bytes::Bytes::from(STREAM_CHUNK_09)), + // Ok(bytes::Bytes::from(STREAM_CHUNK_10)), + ]); + + let actual = string_chunks(&mut source_stream, "\n\n").await?; + let actual: Vec> = actual.collect().await; + + let expected: Vec> = vec![ + Ok(STREAM_EVENT_01.to_string()), + Ok(STREAM_EVENT_02.to_string()), + Ok(STREAM_EVENT_03_01.to_string()), + Ok(STREAM_EVENT_03_02.to_string()), + ]; + + assert_eq!(expected, actual); + Ok(()) + } + + #[tokio::test] + async fn delimiter_search() -> Result<()> { + let delimiter = b"\\n\\n"; + let data = bytes::Bytes::from(STREAM_CHUNK_01); + let mut buffer = Vec::new(); + buffer.extend_from_slice(&data); + + // Find the position of the delimiter + let pos = buffer.windows(4).position(|window| window == delimiter); + match pos { + Some(pos) => { + // it's + 4 because the \n\n are escaped and represented as [92, 110, 92, 110] + let bytes = buffer.drain(..pos).collect::>(); + let yielded_value = std::str::from_utf8(&bytes).unwrap(); + let yielded_value = yielded_value.trim_start_matches("data:").trim(); + + assert_eq!(yielded_value, STREAM_EVENT_01); + } + None => { + println!("Delimiter not found in the buffer"); + assert!(false, "Delimiter not found"); + } + } + + Ok(()) + } + +} diff --git a/sdk/openai_inference/src/clients/tests/mod.rs b/sdk/openai_inference/src/clients/tests/mod.rs new file mode 100644 index 0000000000..5cfa569b8c --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/mod.rs @@ -0,0 +1,19 @@ +pub const STREAM_CHUNK_01: &'static str = r#"data: {"choices":[],"created":0,"id":"","model":"","object":"","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"jailbreak":{"filtered":false,"detected":false},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}]}\n\n"#; +pub const STREAM_EVENT_01: &'static str = r#"{"choices":[],"created":0,"id":"","model":"","object":"","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"jailbreak":{"filtered":false,"detected":false},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}]}"#; + +pub const STREAM_CHUNK_02: &'static str = r#"data: {"choices":[{"content_filter_results":{},"delta":{"role":"assistant"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\n"#; +pub const STREAM_EVENT_02: &'static str = r#"{"choices":[{"content_filter_results":{},"delta":{"role":"assistant"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}"#; + +pub const STREAM_CHUNK_03: &'static str = r#"data: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"This"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" essay"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\n"#; +pub const STREAM_EVENT_03_01: &'static str = r#"{"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"This"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}"#; +pub const STREAM_EVENT_03_02: &'static str = r#"{"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" essay"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}"#; + +pub const STREAM_CHUNK_04: &'static str = include_str!("resources/stream_chunk_04.trace"); +pub const STREAM_CHUNK_05: &'static str = include_str!("resources/stream_chunk_05.trace"); +pub const STREAM_CHUNK_06: &'static str = include_str!("resources/stream_chunk_06.trace"); +pub const STREAM_CHUNK_07: &'static str = include_str!("resources/stream_chunk_07.trace"); +pub const STREAM_CHUNK_08: &'static str = include_str!("resources/stream_chunk_08.trace"); +pub const STREAM_CHUNK_09: &'static str = include_str!("resources/stream_chunk_09.trace"); + +pub const STREAM_CHUNK_10: &'static str = r#"data: [DONE]\n\n"#;//include_str!("resources/stream_chunk_10.trace"); + diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_01.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_01.trace new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_01.trace @@ -0,0 +1 @@ + diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_02.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_02.trace new file mode 100644 index 0000000000..ecfd48b4e6 --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_02.trace @@ -0,0 +1 @@ +data: {"choices":[{"content_filter_results":{},"delta":{"role":"assistant"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\n diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_03.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_03.trace new file mode 100644 index 0000000000..c0684566ee --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_03.trace @@ -0,0 +1 @@ +data: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"This"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" essay"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\n diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_04.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_04.trace new file mode 100644 index 0000000000..e6628fe00b --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_04.trace @@ -0,0 +1,2 @@ + +data: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" will"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" be"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" "},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"48"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" words"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" long"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"."},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" Fast"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" food"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" is"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" often"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" criticized"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" for"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" its"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" poor"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\n diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_05.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_05.trace new file mode 100644 index 0000000000..9115501bd5 --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_05.trace @@ -0,0 +1 @@ +data: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" nutritional"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" value"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" as"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" many"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" items"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" are"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" laden"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" with"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" excessive"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" calories"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" saturated"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" fats"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" sugars"},"finish_reason":null,"index":0}],"created": diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_06.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_06.trace new file mode 100644 index 0000000000..bfe7f3d43f --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_06.trace @@ -0,0 +1 @@ +172323 diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_07.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_07.trace new file mode 100644 index 0000000000..7b5a04b4d5 --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_07.trace @@ -0,0 +1 @@ +3350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" and"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" sodium"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" while"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" lacking"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" essential"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" nutrients"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" like"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" fiber"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" vitamins"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" and"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" minerals"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\n diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_08.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_08.trace new file mode 100644 index 0000000000..aa8470d1ca --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_08.trace @@ -0,0 +1 @@ +data: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"."},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" Regular"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" consumption"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" can"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" lead"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" to"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" obesity"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" heart"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" disease"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":","},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" and"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" other"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" health"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":" problems"},"finish_reason":null,"index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{"custom_blocklists":[],"hate":{"filtered":false,"severity":"safe"},"profanity":{"filtered":false,"detected":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"."},"finish_reason":null,"index":0}],"created":1723233350,"i diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_09.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_09.trace new file mode 100644 index 0000000000..b2fdb2f9ec --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_09.trace @@ -0,0 +1 @@ +d":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\ndata: {"choices":[{"content_filter_results":{},"delta":{},"finish_reason":"stop","index":0}],"created":1723233350,"id":"chatcmpl-9uQ0Ej2jHEoBACZNYUXMaB9WOWDhx","model":"gpt-4","object":"chat.completion.chunk","system_fingerprint":"fp_811936bd4f"}\n\n diff --git a/sdk/openai_inference/src/clients/tests/resources/stream_chunk_10.trace b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_10.trace new file mode 100644 index 0000000000..3ba369451c --- /dev/null +++ b/sdk/openai_inference/src/clients/tests/resources/stream_chunk_10.trace @@ -0,0 +1 @@ +data: [DONE]\n\n diff --git a/sdk/openai_inference/src/lib.rs b/sdk/openai_inference/src/lib.rs new file mode 100644 index 0000000000..ef7717fed1 --- /dev/null +++ b/sdk/openai_inference/src/lib.rs @@ -0,0 +1,8 @@ +mod auth; +mod clients; +mod models; + +pub use crate::auth::{AzureKeyCredential, OpenAIKeyCredential}; +pub use crate::clients::azure_openai::*; +pub use crate::clients::openai::*; +pub use crate::models::*; diff --git a/sdk/openai_inference/src/models/mod.rs b/sdk/openai_inference/src/models/mod.rs new file mode 100644 index 0000000000..e54665b687 --- /dev/null +++ b/sdk/openai_inference/src/models/mod.rs @@ -0,0 +1,7 @@ +mod request; +mod response; + +pub use request::chat_completions::*; +pub use response::chat_completions::*; + +pub use request::audio::*; diff --git a/sdk/openai_inference/src/models/request/audio.rs b/sdk/openai_inference/src/models/request/audio.rs new file mode 100644 index 0000000000..23fefc8fc0 --- /dev/null +++ b/sdk/openai_inference/src/models/request/audio.rs @@ -0,0 +1,47 @@ +#[derive(Debug, Clone, Default)] +pub struct CreateTranscriptionRequest { + pub file: Vec, + pub file_name: String, + pub response_format: OutputFormat, + pub model: Option, +} + +#[derive(Debug, Clone)] +pub struct CreateTranslationRequest { + pub file: Vec, + pub file_name: String, + pub response_format: OutputFormat, +} + +#[derive(Debug, Clone, Default)] +pub enum OutputFormat { + JSON, + #[default] + Text, + SRT, + VerboseJSON, + VTT, +} + +impl ToString for OutputFormat { + fn to_string(&self) -> String { + match self { + OutputFormat::JSON => "json".to_string(), + OutputFormat::Text => "text".to_string(), + OutputFormat::SRT => "srt".to_string(), + OutputFormat::VerboseJSON => "verbose_json".to_string(), + OutputFormat::VTT => "vtt".to_string(), + } + } +} + +impl CreateTranscriptionRequest { + pub fn new_as_text(file: Vec, file_name: impl Into) -> Self { + Self { + file, + file_name: file_name.into(), + model: Some(String::from("whisper-1")), // ignored by azure. TODO: remove. Defaults should be handled better + ..Default::default() + } + } +} diff --git a/sdk/openai_inference/src/models/request/chat_completions.rs b/sdk/openai_inference/src/models/request/chat_completions.rs new file mode 100644 index 0000000000..3af246183a --- /dev/null +++ b/sdk/openai_inference/src/models/request/chat_completions.rs @@ -0,0 +1,66 @@ +use serde::Serialize; + +#[derive(Serialize, Debug, Clone, Default)] +pub struct CreateChatCompletionsRequest { + pub messages: Vec, + pub model: String, + pub stream: Option, + // pub frequency_penalty: f64, + // pub logit_bias: Option>, + // pub logprobs: Option, + // pub top_logprobs: Option, + // pub max_tokens: Option, +} + +#[derive(Serialize, Debug, Clone, Default)] +pub struct ChatCompletionRequestMessageBase { + #[serde(skip)] + pub name: Option, + pub content: String, // TODO this should be either a string or ChatCompletionRequestMessageContentPart (a polymorphic type) +} + +#[derive(Serialize, Debug, Clone)] +#[serde(tag = "role")] +pub enum ChatCompletionRequestMessage { + #[serde(rename = "system")] + System(ChatCompletionRequestMessageBase), + #[serde(rename = "user")] + User(ChatCompletionRequestMessageBase), +} + +impl ChatCompletionRequestMessage { + pub fn new_user(content: impl Into) -> Self { + Self::User(ChatCompletionRequestMessageBase { + content: content.into(), + name: None, + }) + } + + pub fn new_system(content: impl Into) -> Self { + Self::System(ChatCompletionRequestMessageBase { + content: content.into(), + name: None, + }) + } +} +impl CreateChatCompletionsRequest { + pub fn new_with_user_message(model: &str, prompt: &str) -> Self { + Self { + model: model.to_string(), + messages: vec![ChatCompletionRequestMessage::new_user(prompt)], + ..Default::default() + } + } + + pub fn new_stream_with_user_message( + model: impl Into, + prompt: impl Into, + ) -> Self { + Self { + model: model.into(), + messages: vec![ChatCompletionRequestMessage::new_user(prompt)], + stream: Some(true), + ..Default::default() + } + } +} diff --git a/sdk/openai_inference/src/models/request/mod.rs b/sdk/openai_inference/src/models/request/mod.rs new file mode 100644 index 0000000000..d5600318d4 --- /dev/null +++ b/sdk/openai_inference/src/models/request/mod.rs @@ -0,0 +1,2 @@ +pub mod audio; +pub mod chat_completions; diff --git a/sdk/openai_inference/src/models/response/chat_completions.rs b/sdk/openai_inference/src/models/response/chat_completions.rs new file mode 100644 index 0000000000..687fa7dca4 --- /dev/null +++ b/sdk/openai_inference/src/models/response/chat_completions.rs @@ -0,0 +1,36 @@ +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +pub struct CreateChatCompletionsResponse { + pub choices: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ChatCompletionChoice { + pub message: ChatCompletionResponseMessage, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ChatCompletionResponseMessage { + pub content: Option, + pub role: String, +} + +// region: --- Streaming +#[derive(Debug, Clone, Deserialize)] +pub struct CreateChatCompletionsStreamResponse { + pub choices: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ChatCompletionStreamChoice { + pub delta: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ChatCompletionStreamResponseMessage { + pub content: Option, + pub role: Option, +} + +// endregion: Streaming diff --git a/sdk/openai_inference/src/models/response/mod.rs b/sdk/openai_inference/src/models/response/mod.rs new file mode 100644 index 0000000000..dcf3898235 --- /dev/null +++ b/sdk/openai_inference/src/models/response/mod.rs @@ -0,0 +1 @@ +pub mod chat_completions;