-
Notifications
You must be signed in to change notification settings - Fork 627
[FirebaseAI] Add support for Grounding with Google Search #7069
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
4664f6f
5f30a69
c1deb3b
69bd6a2
972f3e5
826c60a
8e835c1
7dbe236
1f42240
80347e0
11d3bde
b26b61a
6b5fa44
04119ad
6e40a19
f9a38a7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -33,13 +33,15 @@ import kotlinx.serialization.json.JsonNames | |
* @property safetyRatings A list of [SafetyRating]s describing the generated content. | ||
* @property citationMetadata Metadata about the sources used to generate this content. | ||
* @property finishReason The reason the model stopped generating content, if it exist. | ||
* @property groundingMetadata Metadata returned to the client when grounding is enabled. | ||
*/ | ||
public class Candidate | ||
internal constructor( | ||
public val content: Content, | ||
public val safetyRatings: List<SafetyRating>, | ||
public val citationMetadata: CitationMetadata?, | ||
public val finishReason: FinishReason? | ||
public val finishReason: FinishReason?, | ||
public val groundingMetadata: GroundingMetadata? | ||
) { | ||
|
||
@Serializable | ||
|
@@ -48,48 +50,22 @@ internal constructor( | |
val finishReason: FinishReason.Internal? = null, | ||
val safetyRatings: List<SafetyRating.Internal>? = null, | ||
val citationMetadata: CitationMetadata.Internal? = null, | ||
val groundingMetadata: GroundingMetadata? = null, | ||
val groundingMetadata: GroundingMetadata.Internal? = null | ||
) { | ||
internal fun toPublic(): Candidate { | ||
val safetyRatings = safetyRatings?.mapNotNull { it.toPublic() }.orEmpty() | ||
val citations = citationMetadata?.toPublic() | ||
val finishReason = finishReason?.toPublic() | ||
val groundingMetadata = groundingMetadata?.toPublic() | ||
|
||
return Candidate( | ||
this.content?.toPublic() ?: content("model") {}, | ||
safetyRatings, | ||
citations, | ||
finishReason | ||
finishReason, | ||
groundingMetadata | ||
) | ||
} | ||
|
||
@Serializable | ||
internal data class GroundingMetadata( | ||
@SerialName("web_search_queries") val webSearchQueries: List<String>?, | ||
@SerialName("search_entry_point") val searchEntryPoint: SearchEntryPoint?, | ||
@SerialName("retrieval_queries") val retrievalQueries: List<String>?, | ||
@SerialName("grounding_attribution") val groundingAttribution: List<GroundingAttribution>?, | ||
) { | ||
|
||
@Serializable | ||
internal data class SearchEntryPoint( | ||
@SerialName("rendered_content") val renderedContent: String?, | ||
@SerialName("sdk_blob") val sdkBlob: String?, | ||
) | ||
|
||
@Serializable | ||
internal data class GroundingAttribution( | ||
val segment: Segment, | ||
@SerialName("confidence_score") val confidenceScore: Float?, | ||
) { | ||
|
||
@Serializable | ||
internal data class Segment( | ||
@SerialName("start_index") val startIndex: Int, | ||
@SerialName("end_index") val endIndex: Int, | ||
) | ||
} | ||
} | ||
} | ||
} | ||
|
||
|
@@ -317,3 +293,203 @@ public class FinishReason private constructor(public val name: String, public va | |
public val MALFORMED_FUNCTION_CALL: FinishReason = FinishReason("MALFORMED_FUNCTION_CALL", 9) | ||
} | ||
} | ||
|
||
/** | ||
* Metadata returned to the client when grounding is enabled. | ||
* | ||
* If using Grounding with Google Search, you are required to comply with the "Grounding with Google | ||
* Search" usage requirements for your chosen API provider: | ||
* [Gemini Developer | ||
* API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or | ||
* Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section | ||
* within the Service Specific Terms). | ||
* | ||
* @property webSearchQueries The list of web search queries that the model performed to gather the | ||
* grounding information. These can be used to allow users to explore the search results themselves. | ||
* @property searchEntryPoint Google search entry point for web searches. This contains an HTML/CSS | ||
* snippet that **must** be embedded in an app to display a Google Search Entry point for follow-up | ||
* web searches related to the model's "Grounded Response". | ||
* @property groundingChunks The list of [GroundingChunk] classes. Each chunk represents a piece of | ||
* retrieved content that the model used to ground its response. | ||
* @property groundingSupports The list of [GroundingSupport] objects. Each object details how | ||
* specific segments of the model's response are supported by the `groundingChunks`. | ||
*/ | ||
public class GroundingMetadata( | ||
public val webSearchQueries: List<String>, | ||
public val searchEntryPoint: SearchEntryPoint?, | ||
public val retrievalQueries: List<String>, | ||
@Deprecated("Use groundingChunks instead") | ||
public val groundingAttribution: List<GroundingAttribution>, | ||
public val groundingChunks: List<GroundingChunk>, | ||
public val groundingSupports: List<GroundingSupport>, | ||
) { | ||
@Serializable | ||
internal data class Internal( | ||
val webSearchQueries: List<String>?, | ||
val searchEntryPoint: SearchEntryPoint.Internal?, | ||
val retrievalQueries: List<String>?, | ||
@Deprecated("Use groundingChunks instead") | ||
val groundingAttribution: List<GroundingAttribution.Internal>?, | ||
val groundingChunks: List<GroundingChunk.Internal>?, | ||
val groundingSupports: List<GroundingSupport.Internal>?, | ||
) { | ||
internal fun toPublic() = | ||
GroundingMetadata( | ||
webSearchQueries = webSearchQueries.orEmpty(), | ||
searchEntryPoint = searchEntryPoint?.toPublic(), | ||
retrievalQueries = retrievalQueries.orEmpty(), | ||
groundingAttribution = groundingAttribution?.map { it.toPublic() }.orEmpty(), | ||
groundingChunks = groundingChunks?.map { it.toPublic() }.orEmpty(), | ||
groundingSupports = groundingSupports?.map { it.toPublic() }.orEmpty().filterNotNull() | ||
) | ||
} | ||
} | ||
|
||
/** | ||
* Represents a Google Search entry point. | ||
* | ||
* @property renderedContent An HTML/CSS snippet that can be embedded in your app. To ensure proper | ||
* rendering, it's recommended to display this content within a `WebView`. | ||
* @property sdkBlob A blob of data for the client SDK to render the search entry point. | ||
*/ | ||
public class SearchEntryPoint( | ||
public val renderedContent: String, | ||
public val sdkBlob: String?, | ||
) { | ||
@Serializable | ||
internal data class Internal( | ||
val renderedContent: String?, | ||
val sdkBlob: String?, | ||
) { | ||
internal fun toPublic(): SearchEntryPoint { | ||
// If rendered content is null, the user must not display the grounded result. If they do, | ||
// they violate the service terms. To prevent this from happening, throw an exception. | ||
if (renderedContent == null) { | ||
throw SerializationException("renderedContent is null, should be a string") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @rlazo Is this the right exception to throw? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This seems like a very interesting case, Why would this happen? Do you have more context here?
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. context: firebase/firebase-ios-sdk#15014 (comment) We expect the server to always respond with
If this is an intentional response from the server, and we don't populate
I can't know what would trigger this, since I have not seen it happen. This is just a precaution in case of a bug in the backend, or if this API evolves in the future, and There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Makes sense. Let's something along the lines of "Discarded response due to missing |
||
} | ||
return SearchEntryPoint(renderedContent = renderedContent, sdkBlob = sdkBlob) | ||
} | ||
} | ||
} | ||
|
||
/** | ||
* Represents a chunk of retrieved data that supports a claim in the model's response. This is part | ||
* of the grounding information provided when grounding is enabled. | ||
* | ||
* @property web Contains details if the grounding chunk is from a web source. | ||
*/ | ||
public class GroundingChunk( | ||
public val web: WebGroundingChunk?, | ||
) { | ||
@Serializable | ||
internal data class Internal( | ||
val web: WebGroundingChunk.Internal?, | ||
) { | ||
internal fun toPublic() = GroundingChunk(web = web?.toPublic()) | ||
} | ||
} | ||
|
||
/** | ||
* A grounding chunk from the web. | ||
* | ||
* @property uri The URI of the retrieved web page. | ||
* @property title The title of the retrieved web page. | ||
* @property domain The domain of the original URI from which the content was retrieved. This is | ||
* only populated when using the Vertex AI Gemini API. | ||
*/ | ||
public class WebGroundingChunk( | ||
public val uri: String?, | ||
public val title: String?, | ||
public val domain: String? | ||
) { | ||
@Serializable | ||
internal data class Internal(val uri: String?, val title: String?, val domain: String?) { | ||
internal fun toPublic() = WebGroundingChunk(uri = uri, title = title, domain = domain) | ||
} | ||
} | ||
|
||
/** | ||
* Provides information about how a specific segment of the model's response is supported by the | ||
* retrieved grounding chunks. | ||
* | ||
* @property segment Specifies the segment of the model's response content that this grounding | ||
* support pertains to. | ||
* @property groundingChunkIndices A list of indices that refer to specific [GroundingChunk] classes | ||
* within the [GroundingMetadata.groundingChunks] array. These referenced chunks are the sources | ||
* that support the claim made in the associated `segment` of the response. For example, an array | ||
* `[1, 3, 4]` means that `groundingChunks[1]`, `groundingChunks[3]`, `groundingChunks[4]` are the | ||
* retrieved content supporting this part of the response. | ||
*/ | ||
public class GroundingSupport( | ||
public val segment: Segment, | ||
public val groundingChunkIndices: List<Int>, | ||
) { | ||
@Serializable | ||
internal data class Internal( | ||
val segment: Segment.Internal?, | ||
val groundingChunkIndices: List<Int>?, | ||
) { | ||
internal fun toPublic(): GroundingSupport? { | ||
if (segment == null) { | ||
return null | ||
} | ||
return GroundingSupport( | ||
segment = segment.toPublic(), | ||
groundingChunkIndices = groundingChunkIndices.orEmpty(), | ||
) | ||
} | ||
} | ||
} | ||
|
||
@Deprecated("Use GroundingChunk instead") | ||
public class GroundingAttribution( | ||
public val segment: Segment, | ||
public val confidenceScore: Float?, | ||
) { | ||
@Deprecated("Use GroundingChunk instead") | ||
@Serializable | ||
internal data class Internal( | ||
val segment: Segment.Internal, | ||
val confidenceScore: Float?, | ||
) { | ||
internal fun toPublic() = | ||
GroundingAttribution(segment = segment.toPublic(), confidenceScore = confidenceScore) | ||
} | ||
} | ||
|
||
/** | ||
* Represents a specific segment within a [Content] object, often used to pinpoint the exact | ||
* location of text or data that grounding information refers to. | ||
* | ||
* @property partIndex The zero-based index of the [Part] object within the `parts` array of its | ||
* parent [Content] object. This identifies which part of the content the segment belongs to. | ||
* @property startIndex The zero-based start index of the segment within the specified [Part], | ||
* measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the beginning of the part's | ||
* content. | ||
* @property endIndex The zero-based end index of the segment within the specified [Part], measured | ||
* in UTF-8 bytes. This offset is exclusive, meaning the character at this index is not included in | ||
* the segment. | ||
* @property text The text corresponding to the segment from the response. | ||
*/ | ||
public class Segment( | ||
public val startIndex: Int, | ||
public val endIndex: Int, | ||
public val partIndex: Int, | ||
public val text: String, | ||
) { | ||
@Serializable | ||
internal data class Internal( | ||
val startIndex: Int?, | ||
val endIndex: Int?, | ||
val partIndex: Int?, | ||
val text: String?, | ||
) { | ||
internal fun toPublic() = | ||
Segment( | ||
startIndex = startIndex ?: 0, | ||
endIndex = endIndex ?: 0, | ||
partIndex = partIndex ?: 0, | ||
text = text ?: "" | ||
) | ||
} | ||
} | ||
dlarocque marked this conversation as resolved.
Show resolved
Hide resolved
|
Uh oh!
There was an error while loading. Please reload this page.