Skip to content

[FirebaseAI] Add support for Grounding with Google Search #7069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions firebase-ai/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
# Unreleased
* [feature] **Breaking Change**: Add support for Grounding with Google Search (#7042).
* **Action Required:** Update all references of `groundingAttributions`, `webSearchQueries`, `retrievalQueries` in `GroundingMetadata` to be non-optional.


# 16.2.0
Expand Down
76 changes: 76 additions & 0 deletions firebase-ai/api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -171,10 +171,12 @@ package com.google.firebase.ai.type {
method public com.google.firebase.ai.type.CitationMetadata? getCitationMetadata();
method public com.google.firebase.ai.type.Content getContent();
method public com.google.firebase.ai.type.FinishReason? getFinishReason();
method public com.google.firebase.ai.type.GroundingMetadata? getGroundingMetadata();
method public java.util.List<com.google.firebase.ai.type.SafetyRating> getSafetyRatings();
property public final com.google.firebase.ai.type.CitationMetadata? citationMetadata;
property public final com.google.firebase.ai.type.Content content;
property public final com.google.firebase.ai.type.FinishReason? finishReason;
property public final com.google.firebase.ai.type.GroundingMetadata? groundingMetadata;
property public final java.util.List<com.google.firebase.ai.type.SafetyRating> safetyRatings;
}

Expand Down Expand Up @@ -398,6 +400,48 @@ package com.google.firebase.ai.type {
method public com.google.firebase.ai.type.GenerativeBackend vertexAI(String location = "us-central1");
}

public final class GoogleSearch {
ctor public GoogleSearch();
}

@Deprecated public final class GroundingAttribution {
ctor @Deprecated public GroundingAttribution(com.google.firebase.ai.type.Segment segment, Float? confidenceScore);
method @Deprecated public Float? getConfidenceScore();
method @Deprecated public com.google.firebase.ai.type.Segment getSegment();
property @Deprecated public final Float? confidenceScore;
property @Deprecated public final com.google.firebase.ai.type.Segment segment;
}

public final class GroundingChunk {
ctor public GroundingChunk(com.google.firebase.ai.type.WebGroundingChunk? web);
method public com.google.firebase.ai.type.WebGroundingChunk? getWeb();
property public final com.google.firebase.ai.type.WebGroundingChunk? web;
}

public final class GroundingMetadata {
ctor public GroundingMetadata(java.util.List<java.lang.String> webSearchQueries, com.google.firebase.ai.type.SearchEntryPoint? searchEntryPoint, java.util.List<java.lang.String> retrievalQueries, @Deprecated java.util.List<com.google.firebase.ai.type.GroundingAttribution> groundingAttribution, java.util.List<com.google.firebase.ai.type.GroundingChunk> groundingChunks, java.util.List<com.google.firebase.ai.type.GroundingSupport> groundingSupports);
method @Deprecated public java.util.List<com.google.firebase.ai.type.GroundingAttribution> getGroundingAttribution();
method public java.util.List<com.google.firebase.ai.type.GroundingChunk> getGroundingChunks();
method public java.util.List<com.google.firebase.ai.type.GroundingSupport> getGroundingSupports();
method public java.util.List<java.lang.String> getRetrievalQueries();
method public com.google.firebase.ai.type.SearchEntryPoint? getSearchEntryPoint();
method public java.util.List<java.lang.String> getWebSearchQueries();
property @Deprecated public final java.util.List<com.google.firebase.ai.type.GroundingAttribution> groundingAttribution;
property public final java.util.List<com.google.firebase.ai.type.GroundingChunk> groundingChunks;
property public final java.util.List<com.google.firebase.ai.type.GroundingSupport> groundingSupports;
property public final java.util.List<java.lang.String> retrievalQueries;
property public final com.google.firebase.ai.type.SearchEntryPoint? searchEntryPoint;
property public final java.util.List<java.lang.String> webSearchQueries;
}

public final class GroundingSupport {
ctor public GroundingSupport(com.google.firebase.ai.type.Segment segment, java.util.List<java.lang.Integer> groundingChunkIndices);
method public java.util.List<java.lang.Integer> getGroundingChunkIndices();
method public com.google.firebase.ai.type.Segment getSegment();
property public final java.util.List<java.lang.Integer> groundingChunkIndices;
property public final com.google.firebase.ai.type.Segment segment;
}

public final class HarmBlockMethod {
method public int getOrdinal();
property public final int ordinal;
Expand Down Expand Up @@ -897,6 +941,26 @@ package com.google.firebase.ai.type {
method public com.google.firebase.ai.type.Schema str(String? description = null, boolean nullable = false, com.google.firebase.ai.type.StringFormat? format = null, String? title = null);
}

public final class SearchEntryPoint {
ctor public SearchEntryPoint(String renderedContent, String? sdkBlob);
method public String getRenderedContent();
method public String? getSdkBlob();
property public final String renderedContent;
property public final String? sdkBlob;
}

public final class Segment {
ctor public Segment(int startIndex, int endIndex, int partIndex, String text);
method public int getEndIndex();
method public int getPartIndex();
method public int getStartIndex();
method public String getText();
property public final int endIndex;
property public final int partIndex;
property public final int startIndex;
property public final String text;
}

public final class SerializationException extends com.google.firebase.ai.type.FirebaseAIException {
}

Expand Down Expand Up @@ -935,11 +999,13 @@ package com.google.firebase.ai.type {

public final class Tool {
method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
field public static final com.google.firebase.ai.type.Tool.Companion Companion;
}

public static final class Tool.Companion {
method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
}

public final class ToolConfig {
Expand Down Expand Up @@ -987,5 +1053,15 @@ package com.google.firebase.ai.type {
@Deprecated public static final class Voices.Companion {
}

public final class WebGroundingChunk {
ctor public WebGroundingChunk(String? uri, String? title, String? domain);
method public String? getDomain();
method public String? getTitle();
method public String? getUri();
property public final String? domain;
property public final String? title;
property public final String? uri;
}

}

238 changes: 207 additions & 31 deletions firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,15 @@ import kotlinx.serialization.json.JsonNames
* @property safetyRatings A list of [SafetyRating]s describing the generated content.
* @property citationMetadata Metadata about the sources used to generate this content.
* @property finishReason The reason the model stopped generating content, if it exist.
* @property groundingMetadata Metadata returned to the client when grounding is enabled.
*/
public class Candidate
internal constructor(
public val content: Content,
public val safetyRatings: List<SafetyRating>,
public val citationMetadata: CitationMetadata?,
public val finishReason: FinishReason?
public val finishReason: FinishReason?,
public val groundingMetadata: GroundingMetadata?
) {

@Serializable
Expand All @@ -48,48 +50,22 @@ internal constructor(
val finishReason: FinishReason.Internal? = null,
val safetyRatings: List<SafetyRating.Internal>? = null,
val citationMetadata: CitationMetadata.Internal? = null,
val groundingMetadata: GroundingMetadata? = null,
val groundingMetadata: GroundingMetadata.Internal? = null
) {
internal fun toPublic(): Candidate {
val safetyRatings = safetyRatings?.mapNotNull { it.toPublic() }.orEmpty()
val citations = citationMetadata?.toPublic()
val finishReason = finishReason?.toPublic()
val groundingMetadata = groundingMetadata?.toPublic()

return Candidate(
this.content?.toPublic() ?: content("model") {},
safetyRatings,
citations,
finishReason
finishReason,
groundingMetadata
)
}

@Serializable
internal data class GroundingMetadata(
@SerialName("web_search_queries") val webSearchQueries: List<String>?,
@SerialName("search_entry_point") val searchEntryPoint: SearchEntryPoint?,
@SerialName("retrieval_queries") val retrievalQueries: List<String>?,
@SerialName("grounding_attribution") val groundingAttribution: List<GroundingAttribution>?,
) {

@Serializable
internal data class SearchEntryPoint(
@SerialName("rendered_content") val renderedContent: String?,
@SerialName("sdk_blob") val sdkBlob: String?,
)

@Serializable
internal data class GroundingAttribution(
val segment: Segment,
@SerialName("confidence_score") val confidenceScore: Float?,
) {

@Serializable
internal data class Segment(
@SerialName("start_index") val startIndex: Int,
@SerialName("end_index") val endIndex: Int,
)
}
}
}
}

Expand Down Expand Up @@ -317,3 +293,203 @@ public class FinishReason private constructor(public val name: String, public va
public val MALFORMED_FUNCTION_CALL: FinishReason = FinishReason("MALFORMED_FUNCTION_CALL", 9)
}
}

/**
* Metadata returned to the client when grounding is enabled.
*
* If using Grounding with Google Search, you are required to comply with the "Grounding with Google
* Search" usage requirements for your chosen API provider:
* [Gemini Developer
* API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or
* Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section
* within the Service Specific Terms).
*
* @property webSearchQueries The list of web search queries that the model performed to gather the
* grounding information. These can be used to allow users to explore the search results themselves.
* @property searchEntryPoint Google search entry point for web searches. This contains an HTML/CSS
* snippet that **must** be embedded in an app to display a Google Search Entry point for follow-up
* web searches related to the model's "Grounded Response".
* @property groundingChunks The list of [GroundingChunk] classes. Each chunk represents a piece of
* retrieved content that the model used to ground its response.
* @property groundingSupports The list of [GroundingSupport] objects. Each object details how
* specific segments of the model's response are supported by the `groundingChunks`.
*/
public class GroundingMetadata(
public val webSearchQueries: List<String>,
public val searchEntryPoint: SearchEntryPoint?,
public val retrievalQueries: List<String>,
@Deprecated("Use groundingChunks instead")
public val groundingAttribution: List<GroundingAttribution>,
public val groundingChunks: List<GroundingChunk>,
public val groundingSupports: List<GroundingSupport>,
) {
@Serializable
internal data class Internal(
val webSearchQueries: List<String>?,
val searchEntryPoint: SearchEntryPoint.Internal?,
val retrievalQueries: List<String>?,
@Deprecated("Use groundingChunks instead")
val groundingAttribution: List<GroundingAttribution.Internal>?,
val groundingChunks: List<GroundingChunk.Internal>?,
val groundingSupports: List<GroundingSupport.Internal>?,
) {
internal fun toPublic() =
GroundingMetadata(
webSearchQueries = webSearchQueries.orEmpty(),
searchEntryPoint = searchEntryPoint?.toPublic(),
retrievalQueries = retrievalQueries.orEmpty(),
groundingAttribution = groundingAttribution?.map { it.toPublic() }.orEmpty(),
groundingChunks = groundingChunks?.map { it.toPublic() }.orEmpty(),
groundingSupports = groundingSupports?.map { it.toPublic() }.orEmpty().filterNotNull()
)
}
}

/**
* Represents a Google Search entry point.
*
* @property renderedContent An HTML/CSS snippet that can be embedded in your app. To ensure proper
* rendering, it's recommended to display this content within a `WebView`.
* @property sdkBlob A blob of data for the client SDK to render the search entry point.
*/
public class SearchEntryPoint(
public val renderedContent: String,
public val sdkBlob: String?,
) {
@Serializable
internal data class Internal(
val renderedContent: String?,
val sdkBlob: String?,
) {
internal fun toPublic(): SearchEntryPoint {
// If rendered content is null, the user must not display the grounded result. If they do,
// they violate the service terms. To prevent this from happening, throw an exception.
if (renderedContent == null) {
throw SerializationException("renderedContent is null, should be a string")
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rlazo Is this the right exception to throw?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems like a very interesting case, Why would this happen? Do you have more context here?

  • If this is an intentional response from the server, would it be reasonable to bubble this up and make the returned object minus grounding result and just log?
  • If this is an error from the server, what triggered it? Would it be better to use that other concept instead of serialization?

Copy link
Author

@dlarocque dlarocque Jun 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

context: firebase/firebase-ios-sdk#15014 (comment)

We expect the server to always respond with renderedContent for a grounded result.

If this is an intentional response from the server, would it be reasonable to bubble this up and make the returned object minus grounding result and just log?

If this is an intentional response from the server, and we don't populate groundingMetadata, the user will still have access to the text, which is a part of the grounded result. Displaying this text without the grounding metadata would be a violation of the service terms. To prevent this from happening, we should not return a grounded response that is impossible to use without violating the service terms.

If this is an error from the server, what triggered it? Would it be better to use that other concept instead of serialization?

I can't know what would trigger this, since I have not seen it happen. This is just a precaution in case of a bug in the backend, or if this API evolves in the future, and renderedContent is deprecated.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense. Let's something along the lines of

"Discarded response due to missing renderedContent."

}
return SearchEntryPoint(renderedContent = renderedContent, sdkBlob = sdkBlob)
}
}
}

/**
* Represents a chunk of retrieved data that supports a claim in the model's response. This is part
* of the grounding information provided when grounding is enabled.
*
* @property web Contains details if the grounding chunk is from a web source.
*/
public class GroundingChunk(
public val web: WebGroundingChunk?,
) {
@Serializable
internal data class Internal(
val web: WebGroundingChunk.Internal?,
) {
internal fun toPublic() = GroundingChunk(web = web?.toPublic())
}
}

/**
* A grounding chunk from the web.
*
* @property uri The URI of the retrieved web page.
* @property title The title of the retrieved web page.
* @property domain The domain of the original URI from which the content was retrieved. This is
* only populated when using the Vertex AI Gemini API.
*/
public class WebGroundingChunk(
public val uri: String?,
public val title: String?,
public val domain: String?
) {
@Serializable
internal data class Internal(val uri: String?, val title: String?, val domain: String?) {
internal fun toPublic() = WebGroundingChunk(uri = uri, title = title, domain = domain)
}
}

/**
* Provides information about how a specific segment of the model's response is supported by the
* retrieved grounding chunks.
*
* @property segment Specifies the segment of the model's response content that this grounding
* support pertains to.
* @property groundingChunkIndices A list of indices that refer to specific [GroundingChunk] classes
* within the [GroundingMetadata.groundingChunks] array. These referenced chunks are the sources
* that support the claim made in the associated `segment` of the response. For example, an array
* `[1, 3, 4]` means that `groundingChunks[1]`, `groundingChunks[3]`, `groundingChunks[4]` are the
* retrieved content supporting this part of the response.
*/
public class GroundingSupport(
public val segment: Segment,
public val groundingChunkIndices: List<Int>,
) {
@Serializable
internal data class Internal(
val segment: Segment.Internal?,
val groundingChunkIndices: List<Int>?,
) {
internal fun toPublic(): GroundingSupport? {
if (segment == null) {
return null
}
return GroundingSupport(
segment = segment.toPublic(),
groundingChunkIndices = groundingChunkIndices.orEmpty(),
)
}
}
}

@Deprecated("Use GroundingChunk instead")
public class GroundingAttribution(
public val segment: Segment,
public val confidenceScore: Float?,
) {
@Deprecated("Use GroundingChunk instead")
@Serializable
internal data class Internal(
val segment: Segment.Internal,
val confidenceScore: Float?,
) {
internal fun toPublic() =
GroundingAttribution(segment = segment.toPublic(), confidenceScore = confidenceScore)
}
}

/**
* Represents a specific segment within a [Content] object, often used to pinpoint the exact
* location of text or data that grounding information refers to.
*
* @property partIndex The zero-based index of the [Part] object within the `parts` array of its
* parent [Content] object. This identifies which part of the content the segment belongs to.
* @property startIndex The zero-based start index of the segment within the specified [Part],
* measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the beginning of the part's
* content.
* @property endIndex The zero-based end index of the segment within the specified [Part], measured
* in UTF-8 bytes. This offset is exclusive, meaning the character at this index is not included in
* the segment.
* @property text The text corresponding to the segment from the response.
*/
public class Segment(
public val startIndex: Int,
public val endIndex: Int,
public val partIndex: Int,
public val text: String,
) {
@Serializable
internal data class Internal(
val startIndex: Int?,
val endIndex: Int?,
val partIndex: Int?,
val text: String?,
) {
internal fun toPublic() =
Segment(
startIndex = startIndex ?: 0,
endIndex = endIndex ?: 0,
partIndex = partIndex ?: 0,
text = text ?: ""
)
}
}
Loading
Loading