@@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest";
44import type { ModelRequests } from "../../accounting/model-requests.ts" ;
55import { effectRunner } from "../../database/database.ts" ;
66import { noDatabase } from "../../database/testing.ts" ;
7+ import type { ProviderIdentity } from "../model-providers/dialects/index.ts" ;
78import { Models } from "./models.ts" ;
89
910/** The fake provider never queries, so nothing here reaches the database. */
@@ -56,8 +57,17 @@ const reply = () =>
5657type Recorded = ModelRequests . Started &
5758 Partial < Omit < ModelRequests . Ending , "outcome" > > & { outcome : ModelRequests . Outcome } ;
5859
59- /** A model over an OpenAI provider that answers with `responses`, one per request. */
60- function modelAnswering ( responses : Array < ( recorded : readonly Recorded [ ] ) => Response > ) {
60+ const OPENAI = {
61+ preset : "openai" ,
62+ baseUrl : "https://api.openai.com/v1" ,
63+ apiFormat : "openai" ,
64+ } satisfies ProviderIdentity ;
65+
66+ /** A model over `provider`, OpenAI's unless given, that answers with `responses`, one per request. */
67+ function modelAnswering (
68+ responses : Array < ( recorded : readonly Recorded [ ] ) => Response > ,
69+ provider : ProviderIdentity = OPENAI ,
70+ ) {
6171 const recorded : Recorded [ ] = [ ] ;
6272 /** Each request's body, as the provider was sent it. */
6373 const sent : Array < Record < string , unknown > > = [ ] ;
@@ -68,9 +78,7 @@ function modelAnswering(responses: Array<(recorded: readonly Recorded[]) => Resp
6878 resolve : ( ) =>
6979 Effect . succeed ( {
7080 providerId : PROVIDER_ID ,
71- preset : "openai" ,
72- baseUrl : "https://api.openai.com/v1" ,
73- apiFormat : "openai" ,
81+ ...provider ,
7482 apiKey : "secret" ,
7583 headers : { } ,
7684 configurationUpdatedAt : new Date ( 0 ) ,
@@ -208,3 +216,75 @@ describe("a response at its step limit", () => {
208216 ] ) ;
209217 } ) ;
210218} ) ;
219+
220+ describe ( "prompt caching" , ( ) => {
221+ const marked = { type : "ephemeral" } ;
222+ const turn : Models . StreamRequest = {
223+ ...input ,
224+ messages : [
225+ { role : "user" , content : "Earlier." } ,
226+ { role : "assistant" , content : "Noted." , cacheBreakpoint : true } ,
227+ { role : "user" , content : "[Turn] Look it up." } ,
228+ ] ,
229+ } ;
230+
231+ it ( "marks the system text, the history's end and each step's newest message for Anthropic" , async ( ) => {
232+ const { model, sent } = modelAnswering ( [ ( ) => new Response ( "Unavailable" , { status : 503 } ) ] , {
233+ preset : "anthropic" ,
234+ baseUrl : "https://api.anthropic.com" ,
235+ apiFormat : "anthropic" ,
236+ } ) ;
237+
238+ await answer ( model , { ...turn , model : "claude-test" } ) ;
239+
240+ expect ( sent [ 0 ] ) . toMatchObject ( {
241+ system : [ { text : input . system , cache_control : marked } ] ,
242+ messages : [
243+ { role : "user" , content : [ { text : "Earlier." } ] } ,
244+ { role : "assistant" , content : [ { text : "Noted." , cache_control : marked } ] } ,
245+ { role : "user" , content : [ { text : "[Turn] Look it up." , cache_control : marked } ] } ,
246+ ] ,
247+ } ) ;
248+ } ) ;
249+
250+ it ( "moves the newest message's mark on each step, keeping the history's" , async ( ) => {
251+ const { model, sent } = modelAnswering ( [ toolCall , reply ] , {
252+ preset : "openrouter" ,
253+ baseUrl : "https://openrouter.ai/api/v1" ,
254+ apiFormat : "openai" ,
255+ } ) ;
256+
257+ await answer ( model , { ...turn , model : "anthropic/claude-test" } ) ;
258+
259+ /** Which of a request's messages are marked, on the message or on one of its parts. */
260+ const markedMessages = ( body : Record < string , unknown > = { } ) =>
261+ ( body . messages as Array < { content : unknown ; cache_control ?: unknown } > ) . map (
262+ ( message ) =>
263+ message . cache_control !== undefined ||
264+ ( Array . isArray ( message . content ) &&
265+ message . content . some ( ( part ) => part . cache_control !== undefined ) ) ,
266+ ) ;
267+ // System text, history, turn.
268+ expect ( markedMessages ( sent [ 0 ] ) ) . toEqual ( [ true , false , true , true ] ) ;
269+ // The turn's mark has moved to the tool result.
270+ expect ( markedMessages ( sent [ 1 ] ) ) . toEqual ( [ true , false , true , false , false , true ] ) ;
271+ } ) ;
272+
273+ it . each ( [
274+ [ "OpenAI" , OPENAI , "gpt-test" ] ,
275+ [
276+ "OpenRouter's other models" ,
277+ { preset : "openrouter" , baseUrl : "https://openrouter.ai/api/v1" , apiFormat : "openai" } ,
278+ "openai/gpt-test" ,
279+ ] ,
280+ ] satisfies Array < [ string , ProviderIdentity , string ] > ) (
281+ "leaves requests to %s unmarked, since they cache by themselves" ,
282+ async ( _name , provider , modelId ) => {
283+ const { model, sent } = modelAnswering ( [ toolCall , reply ] , provider ) ;
284+
285+ await answer ( model , { ...turn , model : modelId } ) ;
286+
287+ expect ( JSON . stringify ( sent ) ) . not . toContain ( "cache_control" ) ;
288+ } ,
289+ ) ;
290+ } ) ;
0 commit comments