Merge remote-tracking branch 'origin/main' into telemetry

Author: andrewbrook

.changeset/spotty-shirts-design.md

@@ -0,0 +1,5 @@
+---
+"@firebase/data-connect": patch
+---
+
+Fixed issue where onComplete wasn't triggering when the user calls `unsubscribe` on a subscription.

common/api-review/ai.api.md

@@ -92,6 +92,10 @@ export interface AudioConversationController {
     stop: () => Promise<void>;
 }
 
+// @public
+export interface AudioTranscriptionConfig {
+}
+
 // @public
 export abstract class Backend {
     protected constructor(type: BackendType);
@@ -153,6 +157,8 @@ export interface ChromeAdapter {
     generateContent(request: GenerateContentRequest): Promise<Response>;
     generateContentStream(request: GenerateContentRequest): Promise<Response>;
     isAvailable(request: GenerateContentRequest): Promise<boolean>;
+    // @internal (undocumented)
+    mode: InferenceMode;
 }
 
 // @public
@@ -256,6 +262,8 @@ export { Date_2 as Date }
 // @public
 export interface EnhancedGenerateContentResponse extends GenerateContentResponse {
     functionCalls: () => FunctionCall[] | undefined;
+    // @beta
+    inferenceSource?: InferenceSource;
     inlineDataParts: () => InlineDataPart[] | undefined;
     text: () => string;
     thoughtSummary: () => string | undefined;
@@ -816,6 +824,15 @@ export const InferenceMode: {
 // @beta
 export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
 
+// @beta
+export const InferenceSource: {
+    readonly ON_DEVICE: "on_device";
+    readonly IN_CLOUD: "in_cloud";
+};
+
+// @beta
+export type InferenceSource = (typeof InferenceSource)[keyof typeof InferenceSource];
+
 // @public
 export interface InlineDataPart {
     // (undocumented)
@@ -911,7 +928,9 @@ export interface LanguageModelPromptOptions {
 // @beta
 export interface LiveGenerationConfig {
     frequencyPenalty?: number;
+    inputAudioTranscription?: AudioTranscriptionConfig;
     maxOutputTokens?: number;
+    outputAudioTranscription?: AudioTranscriptionConfig;
     presencePenalty?: number;
     responseModalities?: ResponseModality[];
     speechConfig?: SpeechConfig;
@@ -964,8 +983,10 @@ export type LiveResponseType = (typeof LiveResponseType)[keyof typeof LiveRespon
 
 // @beta
 export interface LiveServerContent {
+    inputTranscription?: Transcription;
     interrupted?: boolean;
     modelTurn?: Content;
+    outputTranscription?: Transcription;
     turnComplete?: boolean;
     // (undocumented)
     type: 'serverContent';
@@ -994,9 +1015,14 @@ export class LiveSession {
     isClosed: boolean;
     receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
     send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
+    sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
     sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
+    // @deprecated
     sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    // @deprecated (undocumented)
     sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    sendTextRealtime(text: string): Promise<void>;
+    sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
     }
 
 // @public
@@ -1326,6 +1352,11 @@ export interface ToolConfig {
     functionCallingConfig?: FunctionCallingConfig;
 }
 
+// @beta
+export interface Transcription {
+    text?: string;
+}
+
 // @public
 export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema;
 

common/api-review/data-connect.api.md

@@ -109,16 +109,6 @@ export interface DataConnectResult<Data, Variables> extends OpResult<Data> {
     ref: OperationRef<Data, Variables>;
 }
 
-// @public
-export interface DataConnectSubscription<Data, Variables> {
-    // (undocumented)
-    errCallback?: (e?: DataConnectError) => void;
-    // (undocumented)
-    unsubscribe: () => void;
-    // (undocumented)
-    userCallback: OnResultSubscription<Data, Variables>;
-}
-
 // @public (undocumented)
 export type DataSource = typeof SOURCE_CACHE | typeof SOURCE_SERVER;
 

docs-devsite/_toc.yaml

@@ -18,6 +18,8 @@ toc:
     path: /docs/reference/js/ai.arrayschema.md
   - title: AudioConversationController
     path: /docs/reference/js/ai.audioconversationcontroller.md
+  - title: AudioTranscriptionConfig
+    path: /docs/reference/js/ai.audiotranscriptionconfig.md
   - title: Backend
     path: /docs/reference/js/ai.backend.md
   - title: BaseParams
@@ -202,6 +204,8 @@ toc:
     path: /docs/reference/js/ai.thinkingconfig.md
   - title: ToolConfig
     path: /docs/reference/js/ai.toolconfig.md
+  - title: Transcription
+    path: /docs/reference/js/ai.transcription.md
   - title: URLContext
     path: /docs/reference/js/ai.urlcontext.md
   - title: URLContextMetadata

docs-devsite/ai.audiotranscriptionconfig.md

@@ -0,0 +1,19 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# AudioTranscriptionConfig interface
+The audio transcription configuration.
+
+<b>Signature:</b>
+
+```typescript
+export interface AudioTranscriptionConfig 
+```

docs-devsite/ai.enhancedgeneratecontentresponse.md

@@ -24,6 +24,7 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [functionCalls](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () =&gt; [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->\[\] \| undefined | Aggregates and returns every [FunctionCall](./ai.functioncall.md#functioncall_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
+|  [inferenceSource](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinferencesource) | [InferenceSource](./ai.md#inferencesource) | <b><i>(Public Preview)</i></b> Indicates whether inference happened on-device or in-cloud. |
 |  [inlineDataParts](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () =&gt; [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)<!-- -->\[\] \| undefined | Aggregates and returns every [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
 |  [text](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsetext) | () =&gt; string | Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. |
 |  [thoughtSummary](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsethoughtsummary) | () =&gt; string \| undefined | Aggregates and returns every [TextPart](./ai.textpart.md#textpart_interface) with their <code>thought</code> property set to <code>true</code> from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
@@ -38,6 +39,19 @@ Aggregates and returns every [FunctionCall](./ai.functioncall.md#functioncall_in
 functionCalls: () => FunctionCall[] | undefined;
 ```
 
+## EnhancedGenerateContentResponse.inferenceSource
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Indicates whether inference happened on-device or in-cloud.
+
+<b>Signature:</b>
+
+```typescript
+inferenceSource?: InferenceSource;
+```
+
 ## EnhancedGenerateContentResponse.inlineDataParts
 
 Aggregates and returns every [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->.

docs-devsite/ai.livegenerationconfig.md

@@ -26,7 +26,9 @@ export interface LiveGenerationConfig
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [frequencyPenalty](./ai.livegenerationconfig.md#livegenerationconfigfrequencypenalty) | number | <b><i>(Public Preview)</i></b> Frequency penalties. |
+|  [inputAudioTranscription](./ai.livegenerationconfig.md#livegenerationconfiginputaudiotranscription) | [AudioTranscriptionConfig](./ai.audiotranscriptionconfig.md#audiotranscriptionconfig_interface) | <b><i>(Public Preview)</i></b> Enables transcription of audio input.<!-- -->When enabled, the model will respond with transcriptions of your audio input in the <code>inputTranscriptions</code> property in [LiveServerContent](./ai.liveservercontent.md#liveservercontent_interface) messages. Note that the transcriptions are broken up across messages, so you may only receive small amounts of text per message. For example, if you ask the model "How are you today?", the model may transcribe that input across three messages, broken up as "How a", "re yo", "u today?". |
 |  [maxOutputTokens](./ai.livegenerationconfig.md#livegenerationconfigmaxoutputtokens) | number | <b><i>(Public Preview)</i></b> Specifies the maximum number of tokens that can be generated in the response. The number of tokens per word varies depending on the language outputted. Is unbounded by default. |
+|  [outputAudioTranscription](./ai.livegenerationconfig.md#livegenerationconfigoutputaudiotranscription) | [AudioTranscriptionConfig](./ai.audiotranscriptionconfig.md#audiotranscriptionconfig_interface) | <b><i>(Public Preview)</i></b> Enables transcription of audio input.<!-- -->When enabled, the model will respond with transcriptions of its audio output in the <code>outputTranscription</code> property in [LiveServerContent](./ai.liveservercontent.md#liveservercontent_interface) messages. Note that the transcriptions are broken up across messages, so you may only receive small amounts of text per message. For example, if the model says "How are you today?", the model may transcribe that output across three messages, broken up as "How a", "re yo", "u today?". |
 |  [presencePenalty](./ai.livegenerationconfig.md#livegenerationconfigpresencepenalty) | number | <b><i>(Public Preview)</i></b> Positive penalties. |
 |  [responseModalities](./ai.livegenerationconfig.md#livegenerationconfigresponsemodalities) | [ResponseModality](./ai.md#responsemodality)<!-- -->\[\] | <b><i>(Public Preview)</i></b> The modalities of the response. |
 |  [speechConfig](./ai.livegenerationconfig.md#livegenerationconfigspeechconfig) | [SpeechConfig](./ai.speechconfig.md#speechconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for speech synthesis. |
@@ -47,6 +49,21 @@ Frequency penalties.
 frequencyPenalty?: number;
 ```
 
+## LiveGenerationConfig.inputAudioTranscription
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Enables transcription of audio input.
+
+When enabled, the model will respond with transcriptions of your audio input in the `inputTranscriptions` property in [LiveServerContent](./ai.liveservercontent.md#liveservercontent_interface) messages. Note that the transcriptions are broken up across messages, so you may only receive small amounts of text per message. For example, if you ask the model "How are you today?", the model may transcribe that input across three messages, broken up as "How a", "re yo", "u today?".
+
+<b>Signature:</b>
+
+```typescript
+inputAudioTranscription?: AudioTranscriptionConfig;
+```
+
 ## LiveGenerationConfig.maxOutputTokens
 
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
@@ -60,6 +77,21 @@ Specifies the maximum number of tokens that can be generated in the response. Th
 maxOutputTokens?: number;
 ```
 
+## LiveGenerationConfig.outputAudioTranscription
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Enables transcription of audio input.
+
+When enabled, the model will respond with transcriptions of its audio output in the `outputTranscription` property in [LiveServerContent](./ai.liveservercontent.md#liveservercontent_interface) messages. Note that the transcriptions are broken up across messages, so you may only receive small amounts of text per message. For example, if the model says "How are you today?", the model may transcribe that output across three messages, broken up as "How a", "re yo", "u today?".
+
+<b>Signature:</b>
+
+```typescript
+outputAudioTranscription?: AudioTranscriptionConfig;
+```
+
 ## LiveGenerationConfig.presencePenalty
 
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.

docs-devsite/ai.liveservercontent.md

@@ -25,11 +25,26 @@ export interface LiveServerContent
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [inputTranscription](./ai.liveservercontent.md#liveservercontentinputtranscription) | [Transcription](./ai.transcription.md#transcription_interface) | <b><i>(Public Preview)</i></b> Transcription of the audio that was input to the model. |
 |  [interrupted](./ai.liveservercontent.md#liveservercontentinterrupted) | boolean | <b><i>(Public Preview)</i></b> Indicates whether the model was interrupted by the client. An interruption occurs when the client sends a message before the model finishes it's turn. This is <code>undefined</code> if the model was not interrupted. |
 |  [modelTurn](./ai.liveservercontent.md#liveservercontentmodelturn) | [Content](./ai.content.md#content_interface) | <b><i>(Public Preview)</i></b> The content that the model has generated as part of the current conversation with the user. |
+|  [outputTranscription](./ai.liveservercontent.md#liveservercontentoutputtranscription) | [Transcription](./ai.transcription.md#transcription_interface) | <b><i>(Public Preview)</i></b> Transcription of the audio output from the model. |
 |  [turnComplete](./ai.liveservercontent.md#liveservercontentturncomplete) | boolean | <b><i>(Public Preview)</i></b> Indicates whether the turn is complete. This is <code>undefined</code> if the turn is not complete. |
 |  [type](./ai.liveservercontent.md#liveservercontenttype) | 'serverContent' | <b><i>(Public Preview)</i></b> |
 
+## LiveServerContent.inputTranscription
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Transcription of the audio that was input to the model.
+
+<b>Signature:</b>
+
+```typescript
+inputTranscription?: Transcription;
+```
+
 ## LiveServerContent.interrupted
 
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
@@ -56,6 +71,19 @@ The content that the model has generated as part of the current conversation wit
 modelTurn?: Content;
 ```
 
+## LiveServerContent.outputTranscription
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Transcription of the audio output from the model.
+
+<b>Signature:</b>
+
+```typescript
+outputTranscription?: Transcription;
+```
+
 ## LiveServerContent.turnComplete
 
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.